npm - @rocicorp/zero - Versions diffs - 1.9.0-canary.4 → 1.9.0-canary.5 - Mend

@rocicorp/zero 1.9.0-canary.4 → 1.9.0-canary.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (41) hide show

package/out/zero-cache/src/services/change-streamer/storer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#db","#replicaVersion","#onConsumed","#onFatal","#queue","#backPressureThresholdBytes","#statementTimeoutMs","#changeLogBatchSize","#cdc","#approximateQueuedBytes","#running","#readyForMore","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#flushChangeLog","#trackBackfillMetadata","#maybeReleaseBackPressure","#catchup","#upsertTableMetadataStmt","#upsertColumnBackfillStmt","#tx","#released"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {getHeapStatistics} from 'node:v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {\n backfillRequestSchema,\n isDataChange,\n isSchemaChange,\n type BackfillID,\n type BackfillRequest,\n type Change,\n type DataChange,\n type Identifier,\n type SchemaChange,\n type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {\n type ChangeStreamData,\n type Commit,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {\n DownstreamStatusMessage,\n UpstreamStatusMessage,\n} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer-service.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n AutoResetSignal,\n markResetRequired,\n type BackfillingColumn,\n type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n subscriber: Subscriber;\n mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n \| [\n 'change',\n watermark: string,\n json: string,\n orig: Exclude<Change, DataChange> \| null, // null for DataChanges\n ]\n \| ['ready', callback: () => void]\n \| ['subscriber', SubscriberAndMode]\n \| DownstreamStatusMessage\n \| ['abort']\n \| 'stop';\n\ntype PendingTransaction = {\n pool: TransactionPool;\n preCommitWatermark: string;\n pos: number;\n startingReplicationState: Promise<ReplicationOwner>;\n ack: boolean;\n // changeLog rows buffered for the next multi-row INSERT flush.\n batch: ChangeLogRow[];\n // The most recently issued flush (or metadata) process, awaited to bound\n // pipeline depth and to order the commit-time replicationState update.\n lastFlush: Promise<unknown> \| undefined;\n};\n\ntype ReplicationOwner = {\n owner: string \| null;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\nexport type TuningOptions = {\n backPressureLimitHeapProportion: number;\n statementTimeoutMs: number;\n changeLogBatchSize: number;\n};\n\n/*\n A single `changeLog` row, accumulated in {@link PendingTransaction.batch}\n * and written via `json_to_recordset()` (see `#flushChangeLog()`).\n /\ntype ChangeLogRow = {\n watermark: string;\n precommit: string \| null;\n pos: number;\n change: string;\n};\n\n/\n Handles the storage of changes and the catchup of subscribers\n * that are behind.\n \n In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n \n Maintained invariant: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n * `replicaVersion`, which is the version at which initial sync\n * was performed, and any attempts to catchup from a different\n * point fail.\n \n Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must not be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n * a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n * subscriber such that there is a gap in its replication history.\n \n Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n /\nexport class Storer implements Service {\n readonly id = 'storer';\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #taskID: string;\n readonly #discoveryAddress: string;\n readonly #discoveryProtocol: string;\n readonly #db: PostgresDB;\n readonly #replicaVersion: string;\n readonly #onConsumed: (c: Commit \| UpstreamStatusMessage) => void;\n readonly #onFatal: (err: Error) => void;\n readonly #queue = new Queue<QueueEntry>();\n readonly #backPressureThresholdBytes: number;\n readonly #statementTimeoutMs: number;\n readonly #changeLogBatchSize: number;\n\n #approximateQueuedBytes = 0;\n #running = false;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n db: PostgresDB,\n replicaVersion: string,\n onConsumed: (c: Commit \| UpstreamStatusMessage) => void,\n onFatal: (err: Error) => void,\n {\n backPressureLimitHeapProportion,\n statementTimeoutMs,\n changeLogBatchSize,\n }: TuningOptions,\n ) {\n this.#lc = lc.withContext('component', 'change-log');\n this.#shard = shard;\n this.#taskID = taskID;\n this.#discoveryAddress = discoveryAddress;\n this.#discoveryProtocol = discoveryProtocol;\n this.#db = db;\n this.#replicaVersion = replicaVersion;\n this.#onConsumed = onConsumed;\n this.#onFatal = onFatal;\n this.#statementTimeoutMs = statementTimeoutMs;\n this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);\n\n const heapStats = getHeapStatistics();\n this.#backPressureThresholdBytes =\n (heapStats.heap_size_limit - heapStats.used_heap_size) \n backPressureLimitHeapProportion;\n\n this.#lc.info?.(\n `Using up to ${(this.#backPressureThresholdBytes / 1024 2).toFixed(2)} MB of ` +\n `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 2).toFixed(2)} MB) ` +\n `to absorb upstream spikes`,\n {heapStats},\n );\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n async assumeOwnership(purgeLock?: PurgeLock \| null) {\n const db = this.#db;\n const owner = this.#taskID;\n const ownerAddress = this.#discoveryAddress;\n const ownerProtocol = this.#discoveryProtocol;\n // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n const addressWithProtocol =\n ownerProtocol === 'ws'\n ? ownerAddress\n : `${ownerProtocol}://${ownerAddress}`;\n this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n const start = performance.now();\n await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n );\n\n if (purgeLock) {\n // Once ownership has been assumed, any initial purge-lock preventing the\n // purging of change-log records can be released, as a change-streamer\n // that was attempting to purge records will correspondingly abort on the\n // ownership check.\n void purgeLock.release();\n }\n }\n\n async getStartStreamInitializationParameters(): Promise<{\n lastWatermark: string;\n backfillRequests: BackfillRequest[];\n }> {\n const [[{lastWatermark}], result] = await runTx(\n this.#db,\n sql => [\n sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n // Formats a BackfillRequest using json_object_agg() to construct the\n // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n // to make it optional and possibly `null`.\n sql`\n SELECT \n json_build_object(\n 'schema', b.\"schema\",\n 'name', b.\"table\",\n 'metadata', t.\"metadata\"\n ) as \"table\",\n json_object_agg(b.\"column\", b.\"backfill\") \n as \"columns\"\n FROM ${this.#cdc('backfilling')} as b\n LEFT JOIN ${this.#cdc('tableMetadata')} as t\n ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n `,\n ],\n {mode: Mode.READONLY},\n );\n\n return {\n lastWatermark,\n backfillRequests: v.parse(result, backfillRequestsSchema),\n };\n }\n\n async getMinWatermarkForCatchup(): Promise<string \| null> {\n const [{minWatermark}] = await this.#db<{minWatermark: string \| null}[]>\n /sql/ `\n SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n return minWatermark;\n }\n\n purgeRecordsBefore(watermark: string): Promise<number> {\n return runTx(this.#db, async sql => {\n // This NOWAIT pre-check is an optimization to abort the transaction\n // (and release associated resources) early.\n await sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR UPDATE NOWAIT\n `;\n // If the row is purge-locked by an incoming replication-manager, it\n // will assume ownership of the change-log before releasing the lock.\n // This DELETE blocks until the lock is released, allowing the change\n // in ownership to be reliably detected (and the transaction aborted)\n // in the subsequent check.\n const [{deleted}] = await sql<{deleted: bigint}[]>`\n -- The backup watermark can be ahead of the durable changeLog if the\n -- storer is behind but the backup replica has consumed forwarded\n -- changes. Preserve the latest durable changeLog transaction as the\n -- catchup boundary instead of assuming the backup watermark exists.\n -- The storer inserts each changeLog transaction atomically, so any\n -- durable row for a watermark implies the full transaction is durable.\n WITH keep AS (\n SELECT max(watermark) AS watermark\n FROM ${this.#cdc('changeLog')}\n ), purged AS (\n DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n AND watermark < (SELECT watermark FROM keep)\n RETURNING watermark, pos\n ) SELECT COUNT() as deleted FROM purged;`;\n\n const [{owner}] = await sql<ReplicationOwner[]>`\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR SHARE`;\n if (owner !== this.#taskID) {\n throw new AbortError(\n `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n );\n }\n return Number(deleted);\n });\n }\n\n /\n @returns The JSON stringified stream message to be sent downstream.\n /\n store(watermark: string, data: ChangeStreamData) {\n // Eagerly stringify the JSON payload to:\n // - avoid redundant stringification when fanning out to subscribers\n // - efficiently estimate the amount of memory the payload consumes\n const json = BigIntJSON.stringify(data);\n this.#approximateQueuedBytes += json.length;\n\n const change = data[1];\n this.#queue.enqueue([\n 'change',\n watermark,\n json,\n isDataChange(change) ? null : change, // drop DataChanges to save memory\n ]);\n\n return json;\n }\n\n abort() {\n this.#queue.enqueue(['abort']);\n }\n\n status(s: DownstreamStatusMessage) {\n this.#queue.enqueue(s);\n }\n\n catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n }\n\n #readyForMore: Resolver<void> \| null = null;\n\n readyForMore(): Promise<void> \| undefined {\n if (!this.#running) {\n return undefined;\n }\n if (\n this.#readyForMore === null &&\n this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n ) {\n this.#lc.warn?.(\n `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 * 2).toFixed(2)} MB)\\n` +\n `\\n` +\n `To inspect changeLog backlog in your change DB:\\n` +\n ` SELECT\\n` +\n ` (change->'relation'->>'schema') \|\| '.' \|\| (change->'relation'->>'name') AS table_name,\\n` +\n ` change->>'tag' AS operation,\\n` +\n ` COUNT() AS count\\n` +\n ` FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n ` GROUP BY 1, 2\\n` +\n ` ORDER BY 3 DESC\\n` +\n ` LIMIT 20;`,\n );\n this.#readyForMore = resolver();\n }\n return this.#readyForMore?.promise;\n }\n\n #maybeReleaseBackPressure() {\n if (\n this.#readyForMore !== null &&\n // Wait for at least 20% of the threshold to free up.\n this.#approximateQueuedBytes < this.#backPressureThresholdBytes 0.8\n ) {\n this.#lc.info?.(\n `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 2).toFixed(2)} MB)`,\n );\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n }\n\n /\n * Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.\n \n Uses `json_to_recordset()` so the batch is a single JSON parameter: the\n * statement text stays constant regardless of batch size, avoiding the\n * unbounded prepared-statement variants (and Postgres memory growth) that a\n * multi-row INSERT would produce. See rocicorp/mono#3511.\n \n Returns (and updates) {@link PendingTransaction.lastFlush}; a no-op when the\n * batch is empty.\n /\n #flushChangeLog(tx: PendingTransaction): Promise<unknown> \| undefined {\n const {batch} = tx;\n if (batch.length === 0) {\n return tx.lastFlush;\n }\n tx.batch = [];\n tx.lastFlush = tx.pool.process(sql => [\n sql`\n INSERT INTO ${this.#cdc('changeLog')} (\"watermark\", \"pos\", \"change\", \"precommit\")\n SELECT \"watermark\", \"pos\", \"change\"::json, \"precommit\"\n FROM json_to_recordset(${batch}) AS x(\n \"watermark\" TEXT,\n \"pos\" INT8,\n \"change\" TEXT,\n \"precommit\" TEXT\n )`,\n ]);\n return tx.lastFlush;\n }\n\n #stopped = promiseVoid;\n\n /\n Runs the storer loop until {@link stop()} is called, or an error is thrown.\n * Once {@link run()} completes, it can be called again.\n /\n async run() {\n assert(!this.#running, `storer is already running`);\n\n const {promise: stopped, resolve: signalStopped} = resolver();\n this.#running = true;\n this.#stopped = stopped;\n\n this.#lc.info?.('starting storer');\n let err: unknown;\n try {\n await this.#processQueue();\n } catch (e) {\n err = e; // used in finally\n throw e;\n } finally {\n // Release any pending backpressure so the upstream can proceed\n if (this.#readyForMore !== null) {\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n this.#cancelQueueEntries(\n this.#queue.drain().filter(entry => entry !== undefined),\n err,\n );\n this.#running = false;\n signalStopped();\n this.#lc.info?.('storer stopped');\n }\n }\n\n #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n if (queue.length === 0) {\n return;\n }\n this.#lc.info?.(\n `canceling ${queue.length} entries from the changeLog queue`,\n );\n const err = e instanceof Error ? e : new AbortError('server shutting down');\n for (const entry of queue) {\n if (entry === 'stop') {\n continue;\n }\n const type = entry[0];\n switch (type) {\n case 'subscriber': {\n // Disconnect subscribers waiting to be caught up so that they can\n // reconnect and try again.\n const {subscriber} = entry[1];\n this.#lc.info?.(`disconnecting ${subscriber.id}`);\n subscriber.fail(err);\n break;\n }\n }\n }\n }\n\n async #processQueue() {\n let tx: PendingTransaction \| null = null;\n let msg: QueueEntry \| false;\n\n const catchupQueue: SubscriberAndMode[] = [];\n try {\n while ((msg = await this.#queue.dequeue()) !== 'stop') {\n const [msgType] = msg;\n switch (msgType) {\n case 'ready': {\n const signalReady = msg[1];\n signalReady();\n continue;\n }\n case 'subscriber': {\n const subscriber = msg[1];\n if (tx) {\n catchupQueue.push(subscriber); // Wait for the current tx to complete.\n } else {\n await this.#startCatchup([subscriber]); // Catch up immediately.\n }\n continue;\n }\n case 'status':\n this.#onConsumed(msg);\n continue;\n case 'abort': {\n if (tx) {\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n }\n continue;\n }\n }\n // msgType === 'change'\n const [_, watermark, json, change] = msg;\n const tag = change?.tag;\n this.#approximateQueuedBytes -= json.length;\n\n if (tag === 'begin') {\n assert(!tx, 'received BEGIN in the middle of a transaction');\n const {promise, resolve, reject} = resolver<ReplicationOwner>();\n void promise.catch(() => {}); // handle rejections before the await\n tx = {\n pool: new TransactionPool(\n this.#lc.withContext('watermark', watermark),\n {\n mode: Mode.READ_COMMITTED,\n statementResponseTimeout: this.#statementTimeoutMs,\n },\n ),\n preCommitWatermark: watermark,\n pos: 0,\n startingReplicationState: promise,\n ack: !change.skipAck,\n batch: [],\n lastFlush: undefined,\n };\n tx.pool.run(this.#db);\n // Acquire a lock on the replicationState row to detect and/or prevent\n // a concurrent ownership change.\n void tx.pool.process(tx => {\n tx<ReplicationOwner[]> /sql/ `\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n ([result]) => resolve(result),\n reject,\n );\n return [];\n });\n } else {\n assert(tx, () => `received change outside of transaction: ${json}`);\n tx.pos++;\n }\n\n const entry: ChangeLogRow = {\n watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n pos: tx.pos,\n // For backwards compatibility, only the change message is stored\n // in the cdc changeLog.\n change: extractChangeSubstring(json, tag),\n };\n\n if (change !== null && isSchemaChange(change)) {\n // Schema changes carry backfill / table-metadata statements that\n // must be applied in stream order relative to the changeLog rows.\n // Flush any buffered rows first, then write this row together with\n // its metadata statements as a single unit (preserving the previous\n // per-change ordering for schema changes).\n await this.#flushChangeLog(tx);\n tx.lastFlush = tx.pool.process(sql => [\n sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n ...this.#trackBackfillMetadata(sql, change),\n ]);\n } else {\n // Accumulate plain changeLog rows (begin, data changes, commit) and\n // write them as a single multi-row INSERT. Collapsing the per-change\n // single-row INSERTs into batches is the dominant cost reduction for\n // large transactions, where the previous one-statement-per-change\n // path dominated the upstream replication lag.\n tx.batch.push(entry);\n if (tx.batch.length >= this.#changeLogBatchSize) {\n // Bound pipeline depth (and thus memory) by awaiting the previous\n // flush before issuing the next. This is the batched analog of the\n // previous per-100-statement backpressure await, and likewise\n // guards against memory blowup on very large transactions.\n const prevFlush = tx.lastFlush;\n void this.#flushChangeLog(tx);\n await prevFlush;\n }\n }\n this.#maybeReleaseBackPressure();\n\n if (tag === 'commit') {\n // Flush any remaining buffered changeLog rows (including this commit\n // row) before updating the replication state, so the state update is\n // ordered after all changeLog inserts for this transaction.\n void this.#flushChangeLog(tx);\n\n const {owner} = await tx.startingReplicationState;\n if (owner !== this.#taskID) {\n // Ownership change reflected in the replicationState read in 'begin'.\n tx.pool.fail(\n new AbortError(\n `changeLog ownership has been assumed by ${owner}`,\n ),\n );\n } else {\n // Update the replication state.\n const lastWatermark = watermark;\n void tx.pool.process(tx => [\n tx`\n UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n ]);\n tx.pool.setDone();\n }\n\n await tx.pool.done();\n\n // ACK the LSN to the upstream Postgres.\n if (tx.ack) {\n this.#onConsumed(['commit', change, {watermark}]);\n }\n tx = null;\n\n // Before beginning the next transaction, open a READONLY snapshot to\n // concurrently catchup any queued subscribers.\n await this.#startCatchup(catchupQueue.splice(0));\n } else if (tag === 'rollback') {\n // Aborted transactions are not stored in the changeLog. Abort the current tx\n // and process catchup of subscribers that were waiting for it to end.\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n\n await this.#startCatchup(catchupQueue.splice(0));\n }\n }\n } catch (e) {\n catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n }\n\n async #startCatchup(subs: SubscriberAndMode[]) {\n if (subs.length === 0) {\n return;\n }\n\n const reader = new TransactionPool(\n this.#lc.withContext('pool', 'catchup'),\n {mode: Mode.READONLY},\n );\n reader.run(this.#db);\n\n let lastWatermark: string \| undefined;\n try {\n // Ensure that the transaction has started (and is thus holding a snapshot\n // of the database) before continuing on to commit more changes. This is\n // done by performing a single read on the db, which determines the\n // snapshot for the REPEATABLE_READ transaction.\n [{lastWatermark}] = await reader.processReadTask(\n sql => sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}\n `,\n );\n } catch (e) {\n subs.map(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n\n // Run the actual catchup queries in the background. Errors are handled in\n // #catchup() by disconnecting the associated subscriber.\n void Promise.all(\n subs.map(sub => this.#catchup(sub, lastWatermark, reader)),\n ).finally(() => reader.setDone());\n }\n\n async #catchup(\n {subscriber: sub, mode}: SubscriberAndMode,\n lastWatermark: string,\n reader: TransactionPool,\n ) {\n try {\n await reader.processReadTask(async tx => {\n const start = Date.now();\n\n // When starting from initial-sync, there won't be a change with a watermark\n // equal to the replica version. This is the empty changeLog scenario.\n let watermarkFound = sub.watermark === this.#replicaVersion;\n let count = 0;\n let lastBatchConsumed: Promise<unknown> \| undefined;\n\n for await (const entries of tx<ChangeLogEntry[]> /sql/ `\n SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n WHERE watermark >= ${sub.watermark}\n AND watermark <= ${lastWatermark}\n ORDER BY watermark, pos`.cursor(2000)) {\n // Wait for the last batch of entries to be consumed by the\n // subscriber before sending down the current batch. This pipelining\n // allows one batch of changes to be received from the change-db\n // while the previous batch of changes are sent to the subscriber,\n // resulting in flow control that caps the number of changes\n // referenced in memory to 2 batch-size.\n const start = performance.now();\n await lastBatchConsumed;\n const elapsed = performance.now() - start;\n if (lastBatchConsumed) {\n this.#lc[elapsed > 100 ? 'info' : 'debug']?.(\n `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n );\n }\n\n for (const entry of entries) {\n if (entry.watermark === sub.watermark) {\n // This should be the first entry.\n // Catchup starts from after the watermark.\n watermarkFound = true;\n } else if (watermarkFound) {\n lastBatchConsumed = sub.catchup(toDownstream(entry));\n count++;\n } else if (mode === 'backup') {\n throw new AutoResetSignal(\n `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n );\n } else {\n this.#lc.warn?.(\n `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n );\n sub.close(\n ErrorType.WatermarkTooOld,\n `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n );\n return;\n }\n }\n }\n if (watermarkFound) {\n await lastBatchConsumed;\n this.#lc.info?.(\n `caught up ${sub.id} with ${count} changes (${\n Date.now() - start\n } ms)`,\n );\n } else {\n // The subscriber is ahead of the latest durable changeLog entry\n // (lastWatermark). This can legitimately happen: changes are\n // forwarded to subscribers (the backup replica and view-syncers)\n // concurrently with — and can outrun — the durable store, so a\n // replica may briefly lead the change DB after the storer falls\n // behind or the change-streamer restarts. No catchup is possible or\n // needed; once the change DB catches back up, forwarding resumes and\n // the subscriber dedups any watermarks it already has. Unlike the\n // AutoResetSignal / WatermarkTooOld cases above, this is not a gap in\n // replication history, so the subscriber is simply marked caught up.\n this.#lc.warn?.(\n `subscriber ${sub.id} at watermark ${sub.watermark} is ahead of ` +\n `the latest durable watermark ${lastWatermark}; waiting for the ` +\n `change DB to catch up`,\n );\n }\n // Start draining messages buffered during catchup. The returned promise\n // is intentionally not awaited here: while the drain is in progress,\n // new sends keep appending to the subscriber backlog and inherit its\n // byte-based backpressure.\n void sub.setCaughtUp();\n });\n } catch (err) {\n this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n if (err instanceof AutoResetSignal) {\n await markResetRequired(this.#db, this.#shard);\n this.#onFatal(err);\n }\n sub.fail(err);\n }\n }\n\n /*\n Returns the db statements necessary to track backfill and table metadata\n * presented in the `change`, if any.\n /\n #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n const stmts: PendingQuery<Row[]>[] = [];\n\n switch (change.tag) {\n case 'update-table-metadata': {\n const {table, new: metadata} = change;\n stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));\n break;\n }\n\n case 'create-table': {\n const {spec, metadata, backfill} = change;\n if (metadata) {\n stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));\n }\n if (backfill) {\n Object.entries(backfill).forEach(([col, backfill]) => {\n stmts.push(\n this.#upsertColumnBackfillStmt(sql, spec, col, backfill),\n );\n });\n }\n break;\n }\n\n case 'rename-table': {\n const {old} = change;\n const row = {schema: change.new.schema, table: change.new.name};\n stmts.push(\n sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n );\n break;\n }\n\n case 'drop-table': {\n const {\n id: {schema, name},\n } = change;\n stmts.push(\n sql`DELETE FROM ${this.#cdc('tableMetadata')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n );\n break;\n }\n\n case 'add-column': {\n const {table, tableMetadata, column, backfill} = change;\n if (tableMetadata) {\n stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));\n }\n if (backfill) {\n stmts.push(\n this.#upsertColumnBackfillStmt(sql, table, column.name, backfill),\n );\n }\n break;\n }\n\n case 'update-column': {\n const {\n table: {schema, name: table},\n old: {name: oldName},\n new: {name: newName},\n } = change;\n if (oldName !== newName) {\n stmts.push(\n sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${newName}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${oldName}`,\n );\n }\n break;\n }\n\n case 'drop-column': {\n const {\n table: {schema, name},\n column,\n } = change;\n stmts.push(\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${column}`,\n );\n break;\n }\n\n case 'backfill-completed': {\n const {\n relation: {schema, name: table, rowKey},\n columns,\n } = change;\n const cols = [...rowKey.columns, ...columns];\n stmts.push(\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql(cols)}`,\n );\n }\n }\n return stmts;\n }\n\n #upsertTableMetadataStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n metadata: TableMetadata,\n ) {\n const row: TableMetadataRow = {schema, table, metadata};\n return sql`\n INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\") \n DO UPDATE SET ${sql(row)};\n `;\n }\n\n #upsertColumnBackfillStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n column: string,\n backfill: BackfillID,\n ) {\n const row: BackfillingColumn = {schema, table, column, backfill};\n return sql`\n INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\", \"column\") \n DO UPDATE SET ${sql(row)};\n `;\n }\n\n /\n Waits until all currently queued entries have been processed.\n * This is only used in tests.\n /\n async allProcessed() {\n if (this.#running) {\n const {promise, resolve} = resolver();\n this.#queue.enqueue(['ready', resolve]);\n await promise;\n }\n }\n\n stop() {\n if (this.#running) {\n this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n this.#queue.enqueue('stop');\n }\n return this.#stopped;\n }\n}\n\n/\n Extracts the stringified change message from the stringified\n * stream message (e.g. the second tuple element). This optimization\n * facilitates stringifying (and sharing the result of) the stream\n * message exactly once, but storing only the change message substring\n * in the changeLog for backwards compatibility.\n */\nexport function extractChangeSubstring(\n streamMessageJSON: string,\n tag: Change['tag'] \| undefined,\n) {\n switch (tag) {\n case 'begin':\n case 'commit':\n // e.g.\n // [\"begin\",<message-json>,{\"commitWatermark\":\"92fj2d0s\"}]\n // [\"commit\",<message-json>,{\"watermark\":\"92fj2d0s\"}]\n return streamMessageJSON.substring(\n streamMessageJSON.indexOf(',') + 1,\n streamMessageJSON.lastIndexOf(','),\n );\n default:\n // [\"data\",<message-json>]\n return streamMessageJSON.substring(\n streamMessageJSON.indexOf(',') + 1,\n streamMessageJSON.lastIndexOf(']'),\n );\n }\n}\n\ntype ChangeLogEntry = {\n watermark: string;\n tag: string;\n change: string;\n};\n\nfunction toDownstream(entry: ChangeLogEntry): WatermarkedChange {\n const {watermark, change} = entry;\n const tag = entry.tag as ChangeTag;\n switch (tag) {\n case 'begin':\n return [\n watermark,\n tag,\n `[\"begin\",${change},{\"commitWatermark\":\"${watermark}\"}]`,\n ];\n case 'commit':\n return [\n watermark,\n tag,\n `[\"commit\",${change},{\"watermark\":\"${watermark}\"}]`,\n ];\n case 'rollback':\n return [watermark, tag, `[\"rollback\",${change}]`];\n default:\n return [watermark, tag, `[\"data\",${change}]`];\n }\n}\n\nexport class PurgeLock {\n readonly #lc: LogContext;\n readonly #tx: TransactionPool;\n readonly replicaVersion: string;\n readonly minWatermark: string;\n\n constructor(\n lc: LogContext,\n tx: TransactionPool,\n replicaVersion: string,\n watermark: string,\n ) {\n this.#lc = lc;\n this.#tx = tx;\n this.replicaVersion = replicaVersion;\n this.minWatermark = watermark;\n }\n\n #released = false;\n\n async release() {\n if (this.#released) {\n return;\n }\n this.#released = true;\n this.#tx.setDone();\n await this.#tx\n .done()\n .catch(e => this.#lc.warn?.(`error from purge-lock release`, e));\n this.#lc.info?.(`released purge lock on ${this.minWatermark}`);\n }\n}\n\nexport class PurgeLocker {\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #db: PostgresDB;\n\n constructor(lc: LogContext, shard: ShardID, db: PostgresDB) {\n this.#lc = lc.withContext('component', 'purge-locker');\n this.#shard = shard;\n this.#db = db;\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n async acquire() {\n const tx = new TransactionPool(this.#lc, {mode: Mode.READ_COMMITTED}).run(\n this.#db,\n );\n const row = await tx.processReadTask(\n sql => sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR SHARE \n `,\n );\n if (row.length === 0) {\n this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);\n tx.setDone();\n await tx.done();\n return null;\n }\n const [{watermark}] = row;\n const [{replicaVersion}] = await tx.processReadTask(\n sql => sql<{replicaVersion: string}[]>`\n SELECT \"replicaVersion\" FROM ${this.#cdc('replicationConfig')}\n `,\n );\n this.#lc.info?.(\n `locked watermark ${watermark} from being purged from replica@${replicaVersion}`,\n );\n return new PurgeLock(this.#lc, tx, replicaVersion, watermark);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAkFA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiD5D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,MAAkB;CACxC;CACA;CACA;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA,EACE,iCACA,oBACA,sBAEF;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKC,SAAS;EACd,KAAKC,UAAU;EACf,KAAKC,oBAAoB;EACzB,KAAKC,qBAAqB;EAC1B,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,cAAc;EACnB,KAAKC,WAAW;EAChB,KAAKG,sBAAsB;EAC3B,KAAKC,sBAAsB,KAAK,IAAI,GAAG,kBAAkB;EAEzD,MAAM,YAAY,kBAAkB;EACpC,KAAKF,+BACF,UAAU,kBAAkB,UAAU,kBACvC;EAEF,KAAKV,IAAI,OACP,gBAAgB,KAAKU,8BAA8B,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAE/E,EAAC,UAAS,CACZ;CACF;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKL,IAAI,GAAG,UAAU,KAAKJ,MAAM,EAAE,GAAG,OAAO;CACtD;CAEA,MAAM,gBAAgB,WAA8B;EAClD,MAAM,KAAK,KAAKI;EAChB,MAAM,QAAQ,KAAKH;EACnB,MAAM,eAAe,KAAKC;EAC1B,MAAM,gBAAgB,KAAKC;EAE3B,MAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,cAAc,KAAK;EAC5B,KAAKJ,IAAI,OAAO,yBAAyB,qBAAqB;EAC9D,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,EAAE,UAAU,KAAKa,KAAK,kBAAkB,EAAE,OAAO,GAAG;GAAC;GAAO,cAAc;EAAmB,CAAC;EACpG,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,KAAKb,IAAI,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,KAC1D;EAEA,IAAI,WAKF,UAAe,QAAQ;CAE3B;CAEA,MAAM,yCAGH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,UAAU,MAAM,MACxC,KAAKK,MACL,QAAO,CACL,GAA8B;sCACA,KAAKQ,KAAK,kBAAkB,KAK1D,GAAG;;;;;;;;;iBASM,KAAKA,KAAK,aAAa,EAAE;sBACpB,KAAKA,KAAK,eAAe,EAAE;;;SAI3C,GACA,EAAC,MAAM,SAAa,CACtB;EAEA,OAAO;GACL;GACA,kBAAkB,MAAQ,QAAQ,sBAAsB;EAC1D;CACF;CAEA,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,KAAKR,GAC5B;qDACyC,KAAKQ,KAAK,WAAW;EACtE,OAAO;CACT;CAEA,mBAAmB,WAAoC;EACrD,OAAO,MAAM,KAAKR,KAAK,OAAM,QAAO;GAGlC,MAAM,GAA0B;kCACJ,KAAKQ,KAAK,WAAW,EAAE;;;;GASnD,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;;;;;;;;iBASvC,KAAKA,KAAK,WAAW,EAAE;;wBAEhB,KAAKA,KAAK,WAAW,EAAE,qBAAqB,UAAU;;;;GAKxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;8BACvB,KAAKA,KAAK,kBAAkB,EAAE;GACtD,IAAI,UAAU,KAAKX,SACjB,MAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,OAClF;GAEF,OAAO,OAAO,OAAO;EACvB,CAAC;CACH;;;;CAKA,MAAM,WAAmB,MAAwB;EAI/C,MAAM,OAAO,WAAW,UAAU,IAAI;EACtC,KAAKY,2BAA2B,KAAK;EAErC,MAAM,SAAS,KAAK;EACpB,KAAKL,OAAO,QAAQ;GAClB;GACA;GACA;GACA,aAAa,MAAM,IAAI,OAAO;EAChC,CAAC;EAED,OAAO;CACT;CAEA,QAAQ;EACN,KAAKA,OAAO,QAAQ,CAAC,OAAO,CAAC;CAC/B;CAEA,OAAO,GAA4B;EACjC,KAAKA,OAAO,QAAQ,CAAC;CACvB;CAEA,QAAQ,YAAwB,MAAsB;EACpD,KAAKA,OAAO,QAAQ,CAAC,cAAc;GAAC;GAAY;EAAI,CAAC,CAAC;CACxD;CAEA,gBAAuC;CAEvC,eAA0C;EACxC,IAAI,CAAC,KAAKM,UACR;EAEF,IACE,KAAKC,kBAAkB,QACvB,KAAKF,0BAA0B,KAAKJ,6BACpC;GACA,KAAKV,IAAI,OACP,+BAA+B,KAAKS,OAAO,KAAK,EAAE,qBAAqB,KAAKK,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,2SAW9H;GACA,KAAKE,gBAAgB,SAAS;EAChC;EACA,OAAO,KAAKA,eAAe;CAC7B;CAEA,4BAA4B;EAC1B,IACE,KAAKA,kBAAkB,QAEvB,KAAKF,0BAA0B,KAAKJ,8BAA8B,IAClE;GACA,KAAKV,IAAI,OACP,gCAAgC,KAAKS,OAAO,KAAK,EAAE,qBAAqB,KAAKK,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,KAC/H;GACA,KAAKE,cAAc,QAAQ;GAC3B,KAAKA,gBAAgB;EACvB;CACF;;;;;;;;;;;;CAaA,gBAAgB,IAAsD;EACpE,MAAM,EAAC,UAAS;EAChB,IAAI,MAAM,WAAW,GACnB,OAAO,GAAG;EAEZ,GAAG,QAAQ,CAAC;EACZ,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG;sBACa,KAAKH,KAAK,WAAW,EAAE;;mCAEV,MAAM;;;;;YAMrC,CAAC;EACD,OAAO,GAAG;CACZ;CAEA,WAAW;;;;;CAMX,MAAM,MAAM;EACV,OAAO,CAAC,KAAKE,UAAU,2BAA2B;EAElD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,SAAS;EAC5D,KAAKA,WAAW;EAChB,KAAKE,WAAW;EAEhB,KAAKjB,IAAI,OAAO,iBAAiB;EACjC,IAAI;EACJ,IAAI;GACF,MAAM,KAAKkB,cAAc;EAC3B,SAAS,GAAG;GACV,MAAM;GACN,MAAM;EACR,UAAU;GAER,IAAI,KAAKF,kBAAkB,MAAM;IAC/B,KAAKA,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;GACA,KAAKG,oBACH,KAAKV,OAAO,MAAM,EAAE,QAAO,UAAS,UAAU,KAAA,CAAS,GACvD,GACF;GACA,KAAKM,WAAW;GAChB,cAAc;GACd,KAAKf,IAAI,OAAO,gBAAgB;EAClC;CACF;CAEA,oBAAoB,OAAqB,GAAY;EACnD,IAAI,MAAM,WAAW,GACnB;EAEF,KAAKA,IAAI,OACP,aAAa,MAAM,OAAO,kCAC5B;EACA,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,sBAAsB;EAC1E,KAAK,MAAM,SAAS,OAAO;GACzB,IAAI,UAAU,QACZ;GAGF,QADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;KAC3B,KAAKA,IAAI,OAAO,iBAAiB,WAAW,IAAI;KAChD,WAAW,KAAK,GAAG;KACnB;IACF;GACF;EACF;CACF;CAEA,MAAMkB,gBAAgB;EACpB,IAAI,KAAgC;EACpC,IAAI;EAEJ,MAAM,eAAoC,CAAC;EAC3C,IAAI;GACF,QAAQ,MAAM,MAAM,KAAKT,OAAO,QAAQ,OAAO,QAAQ;IACrD,MAAM,CAAC,WAAW;IAClB,QAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;MACxB,YAAY;MACZ;KACF;KACA,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;MACvB,IAAI,IACF,aAAa,KAAK,UAAU;WAE5B,MAAM,KAAKW,cAAc,CAAC,UAAU,CAAC;MAEvC;KACF;KACA,KAAK;MACH,KAAKb,YAAY,GAAG;MACpB;KACF,KAAK;MACH,IAAI,IAAI;OACN,GAAG,KAAK,MAAM;OACd,MAAM,GAAG,KAAK,KAAK;OACnB,KAAK;MACP;MACA;IAEJ;IAEA,MAAM,CAAC,GAAG,WAAW,MAAM,UAAU;IACrC,MAAM,MAAM,QAAQ;IACpB,KAAKO,2BAA2B,KAAK;IAErC,IAAI,QAAQ,SAAS;KACnB,OAAO,CAAC,IAAI,+CAA+C;KAC3D,MAAM,EAAC,SAAS,SAAS,WAAU,SAA2B;KAC9D,QAAa,YAAY,CAAC,CAAC;KAC3B,KAAK;MACH,MAAM,IAAI,gBACR,KAAKd,IAAI,YAAY,aAAa,SAAS,GAC3C;OACE,MAAM;OACN,0BAA0B,KAAKW;MACjC,CACF;MACA,oBAAoB;MACpB,KAAK;MACL,0BAA0B;MAC1B,KAAK,CAAC,OAAO;MACb,OAAO,CAAC;MACR,WAAW,KAAA;KACb;KACA,GAAG,KAAK,IAAI,KAAKN,GAAG;KAGpB,GAAQ,KAAK,SAAQ,OAAM;MACzB,EAA+B;gCACX,KAAKQ,KAAK,kBAAkB,EAAE,aAAa,MAC5D,CAAC,YAAY,QAAQ,MAAM,GAC5B,MACF;MACA,OAAO,CAAC;KACV,CAAC;IACH,OAAO;KACL,OAAO,UAAU,2CAA2C,MAAM;KAClE,GAAG;IACL;IAEA,MAAM,QAAsB;KAC1B,WAAW,QAAQ,WAAW,YAAY,GAAG;KAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;KACtD,KAAK,GAAG;KAGR,QAAQ,uBAAuB,MAAM,GAAG;IAC1C;IAEA,IAAI,WAAW,QAAQ,eAAe,MAAM,GAAG;KAM7C,MAAM,KAAKQ,gBAAgB,EAAE;KAC7B,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG,eAAe,KAAKR,KAAK,WAAW,EAAE,GAAG,IAAI,KAAK,KACrD,GAAG,KAAKS,uBAAuB,KAAK,MAAM,CAC5C,CAAC;IACH,OAAO;KAML,GAAG,MAAM,KAAK,KAAK;KACnB,IAAI,GAAG,MAAM,UAAU,KAAKV,qBAAqB;MAK/C,MAAM,YAAY,GAAG;MACrB,KAAUS,gBAAgB,EAAE;MAC5B,MAAM;KACR;IACF;IACA,KAAKE,0BAA0B;IAE/B,IAAI,QAAQ,UAAU;KAIpB,KAAUF,gBAAgB,EAAE;KAE5B,MAAM,EAAC,UAAS,MAAM,GAAG;KACzB,IAAI,UAAU,KAAKnB,SAEjB,GAAG,KAAK,KACN,IAAI,WACF,2CAA2C,OAC7C,CACF;UACK;MAEL,MAAM,gBAAgB;MACtB,GAAQ,KAAK,SAAQ,OAAM,CACzB,EAAE;qBACK,KAAKW,KAAK,kBAAkB,EAAE,OAAO,GAAG,EAAC,cAAa,CAAC,GAChE,CAAC;MACD,GAAG,KAAK,QAAQ;KAClB;KAEA,MAAM,GAAG,KAAK,KAAK;KAGnB,IAAI,GAAG,KACL,KAAKN,YAAY;MAAC;MAAU;MAAQ,EAAC,UAAS;KAAC,CAAC;KAElD,KAAK;KAIL,MAAM,KAAKa,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,QAAQ,YAAY;KAG7B,GAAG,KAAK,MAAM;KACd,MAAM,GAAG,KAAK,KAAK;KACnB,KAAK;KAEL,MAAM,KAAKA,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD;GACF;EACF,SAAS,GAAG;GACV,aAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GACzD,MAAM;EACR;CACF;CAEA,MAAMA,cAAc,MAA2B;EAC7C,IAAI,KAAK,WAAW,GAClB;EAGF,MAAM,SAAS,IAAI,gBACjB,KAAKpB,IAAI,YAAY,QAAQ,SAAS,GACtC,EAAC,MAAM,SAAa,CACtB;EACA,OAAO,IAAI,KAAKK,GAAG;EAEnB,IAAI;EACJ,IAAI;GAKF,CAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAA8B;sCACP,KAAKQ,KAAK,kBAAkB,EAAE;OAE9D;EACF,SAAS,GAAG;GACV,KAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GAC7C,MAAM;EACR;EAIA,QAAa,IACX,KAAK,KAAI,QAAO,KAAKW,SAAS,KAAK,eAAe,MAAM,CAAC,CAC3D,EAAE,cAAc,OAAO,QAAQ,CAAC;CAClC;CAEA,MAAMA,SACJ,EAAC,YAAY,KAAK,QAClB,eACA,QACA;EACA,IAAI;GACF,MAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,MAAM,QAAQ,KAAK,IAAI;IAIvB,IAAI,iBAAiB,IAAI,cAAc,KAAKlB;IAC5C,IAAI,QAAQ;IACZ,IAAI;IAEJ,WAAW,MAAM,WAAW,EAA6B;sEACK,KAAKO,KAAK,WAAW,EAAE;gCAC7D,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,GAAI,GAAG;KAOxC,MAAM,QAAQ,YAAY,IAAI;KAC9B,MAAM;KACN,MAAM,UAAU,YAAY,IAAI,IAAI;KACpC,IAAI,mBACF,KAAKb,IAAI,UAAU,MAAM,SAAS,WAChC,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,IAAI,GAAG,0CAChD;KAGF,KAAK,MAAM,SAAS,SAClB,IAAI,MAAM,cAAc,IAAI,WAG1B,iBAAiB;UACZ,IAAI,gBAAgB;MACzB,oBAAoB,IAAI,QAAQ,aAAa,KAAK,CAAC;MACnD;KACF,OAAO,IAAI,SAAS,UAClB,MAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,EACvF;UACK;MACL,KAAKA,IAAI,OACP,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,EAC7F;MACA,IAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,EACjF;MACA;KACF;IAEJ;IACA,IAAI,gBAAgB;KAClB,MAAM;KACN,KAAKA,IAAI,OACP,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,IAAI,IAAI,MACd,KACH;IACF,OAWE,KAAKA,IAAI,OACP,cAAc,IAAI,GAAG,gBAAgB,IAAI,UAAU,4CACjB,cAAc,wCAElD;IAMF,IAAS,YAAY;GACvB,CAAC;EACH,SAAS,KAAK;GACZ,KAAKA,IAAI,QAAQ,sCAAsC,IAAI,MAAM,GAAG;GACpE,IAAI,eAAe,iBAAiB;IAClC,MAAM,kBAAkB,KAAKK,KAAK,KAAKJ,MAAM;IAC7C,KAAKO,SAAS,GAAG;GACnB;GACA,IAAI,KAAK,GAAG;EACd;CACF;;;;;CAMA,uBAAuB,KAA0B,QAAsB;EACrE,MAAM,QAA+B,CAAC;EAEtC,QAAQ,OAAO,KAAf;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,KAAK,KAAKiB,yBAAyB,KAAK,OAAO,QAAQ,CAAC;IAC9D;GACF;GAEA,KAAK,gBAAgB;IACnB,MAAM,EAAC,MAAM,UAAU,aAAY;IACnC,IAAI,UACF,MAAM,KAAK,KAAKA,yBAAyB,KAAK,MAAM,QAAQ,CAAC;IAE/D,IAAI,UACF,OAAO,QAAQ,QAAQ,EAAE,SAAS,CAAC,KAAK,cAAc;KACpD,MAAM,KACJ,KAAKC,0BAA0B,KAAK,MAAM,KAAK,QAAQ,CACzD;IACF,CAAC;IAEH;GACF;GAEA,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,OAAO,IAAI;KAAQ,OAAO,OAAO,IAAI;IAAI;IAC9D,MAAM,KACJ,GAAG,UAAU,KAAKb,KAAK,eAAe,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,MAC3D;IACA;GACF;GAEA,KAAK,cAAc;IACjB,MAAM,EACJ,IAAI,EAAC,QAAQ,WACX;IACJ,MAAM,KACJ,GAAG,eAAe,KAAKA,KAAK,eAAe,EAAE;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MACnD;IACA;GACF;GAEA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;IACjD,IAAI,eACF,MAAM,KAAK,KAAKY,yBAAyB,KAAK,OAAO,aAAa,CAAC;IAErE,IAAI,UACF,MAAM,KACJ,KAAKC,0BAA0B,KAAK,OAAO,OAAO,MAAM,QAAQ,CAClE;IAEF;GACF;GAEA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OAAO,EAAC,QAAQ,MAAM,SACtB,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SACd,MAAM,KACJ,GAAG,UAAU,KAAKb,KAAK,aAAa,EAAE,kBAAkB,QAAQ;mCACzC,OAAO,iBAAiB,MAAM,kBAAkB,SACzE;IAEF;GACF;GAEA,KAAK,eAAe;IAClB,MAAM,EACJ,OAAO,EAAC,QAAQ,QAChB,WACE;IACJ,MAAM,KACJ,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,QAC1E;IACA;GACF;GAEA,KAAK,sBAAsB;IACzB,MAAM,EACJ,UAAU,EAAC,QAAQ,MAAM,OAAO,UAChC,YACE;IACJ,MAAM,OAAO,CAAC,GAAG,OAAO,SAAS,GAAG,OAAO;IAC3C,MAAM,KACJ,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,IAAI,GACpF;GACF;EACF;EACA,OAAO;CACT;CAEA,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;EAAQ;EACtD,OAAO,GAAG;sBACQ,KAAKA,KAAK,eAAe,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEnC,IAAI,GAAG,EAAE;;CAEjC;CAEA,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;EAAQ;EAC/D,OAAO,GAAG;sBACQ,KAAKA,KAAK,aAAa,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEjC,IAAI,GAAG,EAAE;;CAEjC;;;;;CAMA,MAAM,eAAe;EACnB,IAAI,KAAKE,UAAU;GACjB,MAAM,EAAC,SAAS,YAAW,SAAS;GACpC,KAAKN,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;GACtC,MAAM;EACR;CACF;CAEA,OAAO;EACL,IAAI,KAAKM,UAAU;GACjB,KAAKf,IAAI,OAAO,YAAY,KAAKS,OAAO,KAAK,EAAE,mBAAmB;GAClE,KAAKA,OAAO,QAAQ,MAAM;EAC5B;EACA,OAAO,KAAKQ;CACd;AACF;;;;;;;;AASA,SAAgB,uBACd,mBACA,KACA;CACA,QAAQ,KAAR;EACE,KAAK;EACL,KAAK,UAIH,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;EACF,SAEE,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;CACJ;AACF;AAQA,SAAS,aAAa,OAA0C;CAC9D,MAAM,EAAC,WAAW,WAAU;CAC5B,MAAM,MAAM,MAAM;CAClB,QAAQ,KAAR;EACE,KAAK,SACH,OAAO;GACL;GACA;GACA,YAAY,OAAO,uBAAuB,UAAU;EACtD;EACF,KAAK,UACH,OAAO;GACL;GACA;GACA,aAAa,OAAO,iBAAiB,UAAU;EACjD;EACF,KAAK,YACH,OAAO;GAAC;GAAW;GAAK,eAAe,OAAO;EAAE;EAClD,SACE,OAAO;GAAC;GAAW;GAAK,WAAW,OAAO;EAAE;CAChD;AACF;AAEA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,gBACA,WACA;EACA,KAAKjB,MAAM;EACX,KAAK2B,MAAM;EACX,KAAK,iBAAiB;EACtB,KAAK,eAAe;CACtB;CAEA,YAAY;CAEZ,MAAM,UAAU;EACd,IAAI,KAAKC,WACP;EAEF,KAAKA,YAAY;EACjB,KAAKD,IAAI,QAAQ;EACjB,MAAM,KAAKA,IACR,KAAK,EACL,OAAM,MAAK,KAAK3B,IAAI,OAAO,iCAAiC,CAAC,CAAC;EACjE,KAAKA,IAAI,OAAO,0BAA0B,KAAK,cAAc;CAC/D;AACF;AAEA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB,IAAgB;EAC1D,KAAKA,MAAM,GAAG,YAAY,aAAa,cAAc;EACrD,KAAKC,SAAS;EACd,KAAKI,MAAM;CACb;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKA,IAAI,GAAG,UAAU,KAAKJ,MAAM,EAAE,GAAG,OAAO;CACtD;CAEA,MAAM,UAAU;EACd,MAAM,KAAK,IAAI,gBAAgB,KAAKD,KAAK,EAAC,MAAM,eAAmB,CAAC,EAAE,IACpE,KAAKK,GACP;EACA,MAAM,MAAM,MAAM,GAAG,iBACnB,QAAO,GAA0B;8BACT,KAAKQ,KAAK,WAAW,EAAE;;;KAIjD;EACA,IAAI,IAAI,WAAW,GAAG;GACpB,KAAKb,IAAI,OAAO,4CAA4C;GAC5D,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,MAAM,CAAC,EAAC,eAAc;EACtB,MAAM,CAAC,EAAC,oBAAmB,MAAM,GAAG,iBAClC,QAAO,GAA+B;uCACL,KAAKa,KAAK,mBAAmB,EAAE;OAElE;EACA,KAAKb,IAAI,OACP,oBAAoB,UAAU,kCAAkC,gBAClE;EACA,OAAO,IAAI,UAAU,KAAKA,KAAK,IAAI,gBAAgB,SAAS;CAC9D;AACF"}
1	+ {"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#db","#replicaVersion","#onConsumed","#onFatal","#queue","#backPressureThresholdBytes","#statementTimeoutMs","#changeLogBatchSize","#drainTimeoutMs","#cdc","#approximateQueuedBytes","#running","#readyForMore","#resetBackpressureTimeout","#backpressureTimeout","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#flushChangeLog","#trackBackfillMetadata","#maybeReleaseBackPressure","#catchup","#upsertTableMetadataStmt","#upsertColumnBackfillStmt","#tx","#released"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {getHeapStatistics} from 'node:v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {orTimeout} from '../../types/timeout.ts';\nimport {\n backfillRequestSchema,\n isDataChange,\n isSchemaChange,\n type BackfillID,\n type BackfillRequest,\n type Change,\n type DataChange,\n type Identifier,\n type SchemaChange,\n type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {\n type ChangeStreamData,\n type Commit,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {\n DownstreamStatusMessage,\n UpstreamStatusMessage,\n} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer-service.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n AutoResetSignal,\n markResetRequired,\n type BackfillingColumn,\n type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n subscriber: Subscriber;\n mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n \| [\n 'change',\n watermark: string,\n json: string,\n orig: Exclude<Change, DataChange> \| null, // null for DataChanges\n ]\n \| ['ready', callback: () => void]\n \| ['subscriber', SubscriberAndMode]\n \| DownstreamStatusMessage\n \| ['abort']\n \| 'stop';\n\ntype PendingTransaction = {\n pool: TransactionPool;\n preCommitWatermark: string;\n pos: number;\n startingReplicationState: Promise<ReplicationOwner>;\n ack: boolean;\n // changeLog rows buffered for the next multi-row INSERT flush.\n batch: ChangeLogRow[];\n // The most recently issued flush (or metadata) process, awaited to bound\n // pipeline depth and to order the commit-time replicationState update.\n lastFlush: Promise<unknown> \| undefined;\n};\n\ntype ReplicationOwner = {\n owner: string \| null;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\nexport type TuningOptions = {\n backPressureLimitHeapProportion: number;\n statementTimeoutMs: number;\n changeLogBatchSize: number;\n drainTimeoutMs?: number \| undefined;\n};\n\n/*\n A single `changeLog` row, accumulated in {@link PendingTransaction.batch}\n * and written via `json_to_recordset()` (see `#flushChangeLog()`).\n /\ntype ChangeLogRow = {\n watermark: string;\n precommit: string \| null;\n pos: number;\n change: string;\n};\n\n/\n Handles the storage of changes and the catchup of subscribers\n * that are behind.\n \n In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n \n Maintained invariant: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n * `replicaVersion`, which is the version at which initial sync\n * was performed, and any attempts to catchup from a different\n * point fail.\n \n Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must not be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n * a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n * subscriber such that there is a gap in its replication history.\n \n Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n /\nexport class Storer implements Service {\n readonly id = 'storer';\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #taskID: string;\n readonly #discoveryAddress: string;\n readonly #discoveryProtocol: string;\n readonly #db: PostgresDB;\n readonly #replicaVersion: string;\n readonly #onConsumed: (c: Commit \| UpstreamStatusMessage) => void;\n readonly #onFatal: (err: Error) => void;\n readonly #queue = new Queue<QueueEntry>();\n readonly #backPressureThresholdBytes: number;\n readonly #statementTimeoutMs: number;\n readonly #changeLogBatchSize: number;\n readonly #drainTimeoutMs: number;\n\n #approximateQueuedBytes = 0;\n #running = false;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n db: PostgresDB,\n replicaVersion: string,\n onConsumed: (c: Commit \| UpstreamStatusMessage) => void,\n onFatal: (err: Error) => void,\n {\n backPressureLimitHeapProportion,\n statementTimeoutMs,\n changeLogBatchSize,\n drainTimeoutMs = 30_000,\n }: TuningOptions,\n ) {\n this.#lc = lc.withContext('component', 'change-log');\n this.#shard = shard;\n this.#taskID = taskID;\n this.#discoveryAddress = discoveryAddress;\n this.#discoveryProtocol = discoveryProtocol;\n this.#db = db;\n this.#replicaVersion = replicaVersion;\n this.#onConsumed = onConsumed;\n this.#onFatal = onFatal;\n this.#statementTimeoutMs = statementTimeoutMs;\n this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);\n this.#drainTimeoutMs = drainTimeoutMs;\n\n const heapStats = getHeapStatistics();\n this.#backPressureThresholdBytes =\n (heapStats.heap_size_limit - heapStats.used_heap_size) \n backPressureLimitHeapProportion;\n\n this.#lc.info?.(\n `Using up to ${(this.#backPressureThresholdBytes / 1024 2).toFixed(2)} MB of ` +\n `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 2).toFixed(2)} MB) ` +\n `to absorb upstream spikes`,\n {heapStats},\n );\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n async assumeOwnership(purgeLock?: PurgeLock \| null) {\n const db = this.#db;\n const owner = this.#taskID;\n const ownerAddress = this.#discoveryAddress;\n const ownerProtocol = this.#discoveryProtocol;\n // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n const addressWithProtocol =\n ownerProtocol === 'ws'\n ? ownerAddress\n : `${ownerProtocol}://${ownerAddress}`;\n this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n const start = performance.now();\n await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n );\n\n if (purgeLock) {\n // Once ownership has been assumed, any initial purge-lock preventing the\n // purging of change-log records can be released, as a change-streamer\n // that was attempting to purge records will correspondingly abort on the\n // ownership check.\n void purgeLock.release();\n }\n }\n\n async getStartStreamInitializationParameters(): Promise<{\n lastWatermark: string;\n backfillRequests: BackfillRequest[];\n }> {\n const [[{lastWatermark}], result] = await runTx(\n this.#db,\n sql => [\n sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n // Formats a BackfillRequest using json_object_agg() to construct the\n // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n // to make it optional and possibly `null`.\n sql`\n SELECT \n json_build_object(\n 'schema', b.\"schema\",\n 'name', b.\"table\",\n 'metadata', t.\"metadata\"\n ) as \"table\",\n json_object_agg(b.\"column\", b.\"backfill\") \n as \"columns\"\n FROM ${this.#cdc('backfilling')} as b\n LEFT JOIN ${this.#cdc('tableMetadata')} as t\n ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n `,\n ],\n {mode: Mode.READONLY},\n );\n\n return {\n lastWatermark,\n backfillRequests: v.parse(result, backfillRequestsSchema),\n };\n }\n\n async getMinWatermarkForCatchup(): Promise<string \| null> {\n const [{minWatermark}] = await this.#db<{minWatermark: string \| null}[]>\n /sql/ `\n SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n return minWatermark;\n }\n\n purgeRecordsBefore(watermark: string): Promise<number> {\n return runTx(this.#db, async sql => {\n // This NOWAIT pre-check is an optimization to abort the transaction\n // (and release associated resources) early.\n await sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR UPDATE NOWAIT\n `;\n // If the row is purge-locked by an incoming replication-manager, it\n // will assume ownership of the change-log before releasing the lock.\n // This DELETE blocks until the lock is released, allowing the change\n // in ownership to be reliably detected (and the transaction aborted)\n // in the subsequent check.\n const [{deleted}] = await sql<{deleted: bigint}[]>`\n -- The backup watermark can be ahead of the durable changeLog if the\n -- storer is behind but the backup replica has consumed forwarded\n -- changes. Preserve the latest durable changeLog transaction as the\n -- catchup boundary instead of assuming the backup watermark exists.\n -- The storer inserts each changeLog transaction atomically, so any\n -- durable row for a watermark implies the full transaction is durable.\n WITH keep AS (\n SELECT max(watermark) AS watermark\n FROM ${this.#cdc('changeLog')}\n ), purged AS (\n DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n AND watermark < (SELECT watermark FROM keep)\n RETURNING watermark, pos\n ) SELECT COUNT() as deleted FROM purged;`;\n\n const [{owner}] = await sql<ReplicationOwner[]>`\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR SHARE`;\n if (owner !== this.#taskID) {\n throw new AbortError(\n `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n );\n }\n return Number(deleted);\n });\n }\n\n /\n @returns The JSON stringified stream message to be sent downstream.\n /\n store(watermark: string, data: ChangeStreamData) {\n // Eagerly stringify the JSON payload to:\n // - avoid redundant stringification when fanning out to subscribers\n // - efficiently estimate the amount of memory the payload consumes\n const json = BigIntJSON.stringify(data);\n this.#approximateQueuedBytes += json.length;\n\n const change = data[1];\n this.#queue.enqueue([\n 'change',\n watermark,\n json,\n isDataChange(change) ? null : change, // drop DataChanges to save memory\n ]);\n\n return json;\n }\n\n abort() {\n this.#queue.enqueue(['abort']);\n }\n\n status(s: DownstreamStatusMessage) {\n this.#queue.enqueue(s);\n }\n\n catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n }\n\n #readyForMore: Resolver<void> \| null = null;\n #backpressureTimeout: NodeJS.Timeout \| undefined;\n\n readyForMore(): Promise<void> \| undefined {\n if (!this.#running) {\n return undefined;\n }\n if (\n this.#readyForMore === null &&\n this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n ) {\n this.#lc.warn?.(\n `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 * 2).toFixed(2)} MB)\\n` +\n `\\n` +\n `To inspect changeLog backlog in your change DB:\\n` +\n ` SELECT\\n` +\n ` (change->'relation'->>'schema') \|\| '.' \|\| (change->'relation'->>'name') AS table_name,\\n` +\n ` change->>'tag' AS operation,\\n` +\n ` COUNT() AS count\\n` +\n ` FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n ` GROUP BY 1, 2\\n` +\n ` ORDER BY 3 DESC\\n` +\n ` LIMIT 20;`,\n );\n this.#readyForMore = resolver();\n this.#resetBackpressureTimeout(true);\n }\n return this.#readyForMore?.promise;\n }\n\n #resetBackpressureTimeout(reschedule: boolean) {\n clearTimeout(this.#backpressureTimeout);\n this.#backpressureTimeout = reschedule\n ? setTimeout(() => {\n this.#readyForMore?.reject(\n new AbortError(\n `No statement processed within the last ${this.#statementTimeoutMs}ms. Considering the change-log to be wedged.`,\n ),\n );\n this.#readyForMore = null;\n }, this.#statementTimeoutMs)\n : undefined;\n }\n\n #maybeReleaseBackPressure() {\n if (this.#readyForMore !== null) {\n // Wait for at least 20% of the threshold to free up.\n if (\n this.#approximateQueuedBytes <\n this.#backPressureThresholdBytes 0.8\n ) {\n this.#lc.info?.(\n `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 2).toFixed(2)} MB)`,\n );\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n this.#resetBackpressureTimeout(false);\n } else {\n this.#resetBackpressureTimeout(true);\n }\n }\n }\n\n /\n * Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.\n \n Uses `json_to_recordset()` so the batch is a single JSON parameter: the\n * statement text stays constant regardless of batch size, avoiding the\n * unbounded prepared-statement variants (and Postgres memory growth) that a\n * multi-row INSERT would produce. See rocicorp/mono#3511.\n \n Returns (and updates) {@link PendingTransaction.lastFlush}; a no-op when the\n * batch is empty.\n /\n #flushChangeLog(tx: PendingTransaction): Promise<unknown> \| undefined {\n const {batch} = tx;\n if (batch.length === 0) {\n return tx.lastFlush;\n }\n tx.batch = [];\n tx.lastFlush = tx.pool.process(sql => [\n sql`\n INSERT INTO ${this.#cdc('changeLog')} (\"watermark\", \"pos\", \"change\", \"precommit\")\n SELECT \"watermark\", \"pos\", \"change\"::json, \"precommit\"\n FROM json_to_recordset(${batch}) AS x(\n \"watermark\" TEXT,\n \"pos\" INT8,\n \"change\" TEXT,\n \"precommit\" TEXT\n )`,\n ]);\n return tx.lastFlush;\n }\n\n #stopped = promiseVoid;\n\n /\n Runs the storer loop until {@link stop()} is called, or an error is thrown.\n * Once {@link run()} completes, it can be called again.\n /\n async run() {\n assert(!this.#running, `storer is already running`);\n\n const {promise: stopped, resolve: signalStopped} = resolver();\n this.#running = true;\n this.#stopped = stopped;\n\n this.#lc.info?.('starting storer');\n let err: unknown;\n try {\n await this.#processQueue();\n } catch (e) {\n err = e; // used in finally\n throw e;\n } finally {\n // Release any pending backpressure so the upstream can proceed\n if (this.#readyForMore !== null) {\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n this.#resetBackpressureTimeout(false);\n this.#cancelQueueEntries(\n this.#queue.drain().filter(entry => entry !== undefined),\n err,\n );\n this.#running = false;\n signalStopped();\n this.#lc.info?.('storer stopped');\n }\n }\n\n #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n if (queue.length === 0) {\n return;\n }\n this.#lc.info?.(\n `canceling ${queue.length} entries from the changeLog queue`,\n );\n const err = e instanceof Error ? e : new AbortError('server shutting down');\n for (const entry of queue) {\n if (entry === 'stop') {\n continue;\n }\n const type = entry[0];\n switch (type) {\n case 'subscriber': {\n // Disconnect subscribers waiting to be caught up so that they can\n // reconnect and try again.\n const {subscriber} = entry[1];\n this.#lc.info?.(`disconnecting ${subscriber.id}`);\n subscriber.fail(err);\n break;\n }\n }\n }\n }\n\n async #processQueue() {\n let tx: PendingTransaction \| null = null;\n let msg: QueueEntry \| false;\n\n const catchupQueue: SubscriberAndMode[] = [];\n try {\n while ((msg = await this.#queue.dequeue()) !== 'stop') {\n const [msgType] = msg;\n switch (msgType) {\n case 'ready': {\n const signalReady = msg[1];\n signalReady();\n continue;\n }\n case 'subscriber': {\n const subscriber = msg[1];\n if (tx) {\n catchupQueue.push(subscriber); // Wait for the current tx to complete.\n } else {\n await this.#startCatchup([subscriber]); // Catch up immediately.\n }\n continue;\n }\n case 'status':\n this.#onConsumed(msg);\n continue;\n case 'abort': {\n if (tx) {\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n }\n continue;\n }\n }\n // msgType === 'change'\n const [_, watermark, json, change] = msg;\n const tag = change?.tag;\n this.#approximateQueuedBytes -= json.length;\n\n if (tag === 'begin') {\n assert(!tx, 'received BEGIN in the middle of a transaction');\n const {promise, resolve, reject} = resolver<ReplicationOwner>();\n void promise.catch(() => {}); // handle rejections before the await\n tx = {\n pool: new TransactionPool(\n this.#lc.withContext('watermark', watermark),\n {\n mode: Mode.READ_COMMITTED,\n statementResponseTimeout: this.#statementTimeoutMs,\n },\n ),\n preCommitWatermark: watermark,\n pos: 0,\n startingReplicationState: promise,\n ack: !change.skipAck,\n batch: [],\n lastFlush: undefined,\n };\n tx.pool.run(this.#db);\n // Acquire a lock on the replicationState row to detect and/or prevent\n // a concurrent ownership change.\n void tx.pool.process(tx => {\n tx<ReplicationOwner[]> /sql/ `\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n ([result]) => resolve(result),\n reject,\n );\n return [];\n });\n } else {\n assert(tx, () => `received change outside of transaction: ${json}`);\n tx.pos++;\n }\n\n const entry: ChangeLogRow = {\n watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n pos: tx.pos,\n // For backwards compatibility, only the change message is stored\n // in the cdc changeLog.\n change: extractChangeSubstring(json, tag),\n };\n\n if (change !== null && isSchemaChange(change)) {\n // Schema changes carry backfill / table-metadata statements that\n // must be applied in stream order relative to the changeLog rows.\n // Flush any buffered rows first, then write this row together with\n // its metadata statements as a single unit (preserving the previous\n // per-change ordering for schema changes).\n await this.#flushChangeLog(tx);\n tx.lastFlush = tx.pool.process(sql => [\n sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n ...this.#trackBackfillMetadata(sql, change),\n ]);\n } else {\n // Accumulate plain changeLog rows (begin, data changes, commit) and\n // write them as a single multi-row INSERT. Collapsing the per-change\n // single-row INSERTs into batches is the dominant cost reduction for\n // large transactions, where the previous one-statement-per-change\n // path dominated the upstream replication lag.\n tx.batch.push(entry);\n if (tx.batch.length >= this.#changeLogBatchSize) {\n // Bound pipeline depth (and thus memory) by awaiting the previous\n // flush before issuing the next. This is the batched analog of the\n // previous per-100-statement backpressure await, and likewise\n // guards against memory blowup on very large transactions.\n const prevFlush = tx.lastFlush;\n void this.#flushChangeLog(tx);\n await prevFlush;\n }\n }\n this.#maybeReleaseBackPressure();\n\n if (tag === 'commit') {\n // Flush any remaining buffered changeLog rows (including this commit\n // row) before updating the replication state, so the state update is\n // ordered after all changeLog inserts for this transaction.\n void this.#flushChangeLog(tx);\n\n const {owner} = await tx.startingReplicationState;\n if (owner !== this.#taskID) {\n // Ownership change reflected in the replicationState read in 'begin'.\n tx.pool.fail(\n new AbortError(\n `changeLog ownership has been assumed by ${owner}`,\n ),\n );\n } else {\n // Update the replication state.\n const lastWatermark = watermark;\n void tx.pool.process(tx => [\n tx`\n UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n ]);\n tx.pool.setDone();\n }\n\n await tx.pool.done();\n\n // ACK the LSN to the upstream Postgres.\n if (tx.ack) {\n this.#onConsumed(['commit', change, {watermark}]);\n }\n tx = null;\n\n // Before beginning the next transaction, open a READONLY snapshot to\n // concurrently catchup any queued subscribers.\n await this.#startCatchup(catchupQueue.splice(0));\n } else if (tag === 'rollback') {\n // Aborted transactions are not stored in the changeLog. Abort the current tx\n // and process catchup of subscribers that were waiting for it to end.\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n\n await this.#startCatchup(catchupQueue.splice(0));\n }\n }\n } catch (e) {\n catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n }\n\n async #startCatchup(subs: SubscriberAndMode[]) {\n if (subs.length === 0) {\n return;\n }\n\n const reader = new TransactionPool(\n this.#lc.withContext('pool', 'catchup'),\n {mode: Mode.READONLY},\n );\n reader.run(this.#db);\n\n let lastWatermark: string \| undefined;\n try {\n // Ensure that the transaction has started (and is thus holding a snapshot\n // of the database) before continuing on to commit more changes. This is\n // done by performing a single read on the db, which determines the\n // snapshot for the REPEATABLE_READ transaction.\n [{lastWatermark}] = await reader.processReadTask(\n sql => sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}\n `,\n );\n } catch (e) {\n subs.map(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n\n // Run the actual catchup queries in the background. Errors are handled in\n // #catchup() by disconnecting the associated subscriber.\n void Promise.all(\n subs.map(sub => this.#catchup(sub, lastWatermark, reader)),\n ).finally(() => reader.setDone());\n }\n\n async #catchup(\n {subscriber: sub, mode}: SubscriberAndMode,\n lastWatermark: string,\n reader: TransactionPool,\n ) {\n try {\n await reader.processReadTask(async tx => {\n const start = Date.now();\n\n // When starting from initial-sync, there won't be a change with a watermark\n // equal to the replica version. This is the empty changeLog scenario.\n let watermarkFound = sub.watermark === this.#replicaVersion;\n let count = 0;\n let lastBatchConsumed: Promise<unknown> \| undefined;\n\n for await (const entries of tx<ChangeLogEntry[]> /sql/ `\n SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n WHERE watermark >= ${sub.watermark}\n AND watermark <= ${lastWatermark}\n ORDER BY watermark, pos`.cursor(2000)) {\n // Wait for the last batch of entries to be consumed by the\n // subscriber before sending down the current batch. This pipelining\n // allows one batch of changes to be received from the change-db\n // while the previous batch of changes are sent to the subscriber,\n // resulting in flow control that caps the number of changes\n // referenced in memory to 2 batch-size.\n const start = performance.now();\n await lastBatchConsumed;\n const elapsed = performance.now() - start;\n if (lastBatchConsumed) {\n this.#lc[elapsed > 100 ? 'info' : 'debug']?.(\n `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n );\n }\n\n for (const entry of entries) {\n if (entry.watermark === sub.watermark) {\n // This should be the first entry.\n // Catchup starts from after the watermark.\n watermarkFound = true;\n } else if (watermarkFound) {\n lastBatchConsumed = sub.catchup(toDownstream(entry));\n count++;\n } else if (mode === 'backup') {\n throw new AutoResetSignal(\n `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n );\n } else {\n this.#lc.warn?.(\n `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n );\n sub.close(\n ErrorType.WatermarkTooOld,\n `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n );\n return;\n }\n }\n }\n if (watermarkFound) {\n await lastBatchConsumed;\n this.#lc.info?.(\n `caught up ${sub.id} with ${count} changes (${\n Date.now() - start\n } ms)`,\n );\n } else {\n // The subscriber is ahead of the latest durable changeLog entry\n // (lastWatermark). This can legitimately happen: changes are\n // forwarded to subscribers (the backup replica and view-syncers)\n // concurrently with — and can outrun — the durable store, so a\n // replica may briefly lead the change DB after the storer falls\n // behind or the change-streamer restarts. No catchup is possible or\n // needed; once the change DB catches back up, forwarding resumes and\n // the subscriber dedups any watermarks it already has. Unlike the\n // AutoResetSignal / WatermarkTooOld cases above, this is not a gap in\n // replication history, so the subscriber is simply marked caught up.\n this.#lc.warn?.(\n `subscriber ${sub.id} at watermark ${sub.watermark} is ahead of ` +\n `the latest durable watermark ${lastWatermark}; waiting for the ` +\n `change DB to catch up`,\n );\n }\n // Start draining messages buffered during catchup. The returned promise\n // is intentionally not awaited here: while the drain is in progress,\n // new sends keep appending to the subscriber backlog and inherit its\n // byte-based backpressure.\n void sub.setCaughtUp();\n });\n } catch (err) {\n this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n if (err instanceof AutoResetSignal) {\n await markResetRequired(this.#db, this.#shard);\n this.#onFatal(err);\n }\n sub.fail(err);\n }\n }\n\n /*\n Returns the db statements necessary to track backfill and table metadata\n * presented in the `change`, if any.\n /\n #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n const stmts: PendingQuery<Row[]>[] = [];\n\n switch (change.tag) {\n case 'update-table-metadata': {\n const {table, new: metadata} = change;\n stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));\n break;\n }\n\n case 'create-table': {\n const {spec, metadata, backfill} = change;\n if (metadata) {\n stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));\n }\n if (backfill) {\n Object.entries(backfill).forEach(([col, backfill]) => {\n stmts.push(\n this.#upsertColumnBackfillStmt(sql, spec, col, backfill),\n );\n });\n }\n break;\n }\n\n case 'rename-table': {\n const {old} = change;\n const row = {schema: change.new.schema, table: change.new.name};\n stmts.push(\n sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n );\n break;\n }\n\n case 'drop-table': {\n const {\n id: {schema, name},\n } = change;\n stmts.push(\n sql`DELETE FROM ${this.#cdc('tableMetadata')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n );\n break;\n }\n\n case 'add-column': {\n const {table, tableMetadata, column, backfill} = change;\n if (tableMetadata) {\n stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));\n }\n if (backfill) {\n stmts.push(\n this.#upsertColumnBackfillStmt(sql, table, column.name, backfill),\n );\n }\n break;\n }\n\n case 'update-column': {\n const {\n table: {schema, name: table},\n old: {name: oldName},\n new: {name: newName},\n } = change;\n if (oldName !== newName) {\n stmts.push(\n sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${newName}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${oldName}`,\n );\n }\n break;\n }\n\n case 'drop-column': {\n const {\n table: {schema, name},\n column,\n } = change;\n stmts.push(\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${column}`,\n );\n break;\n }\n\n case 'backfill-completed': {\n const {\n relation: {schema, name: table, rowKey},\n columns,\n } = change;\n const cols = [...rowKey.columns, ...columns];\n stmts.push(\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql(cols)}`,\n );\n }\n }\n return stmts;\n }\n\n #upsertTableMetadataStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n metadata: TableMetadata,\n ) {\n const row: TableMetadataRow = {schema, table, metadata};\n return sql`\n INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\") \n DO UPDATE SET ${sql(row)};\n `;\n }\n\n #upsertColumnBackfillStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n column: string,\n backfill: BackfillID,\n ) {\n const row: BackfillingColumn = {schema, table, column, backfill};\n return sql`\n INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\", \"column\") \n DO UPDATE SET ${sql(row)};\n `;\n }\n\n /\n Waits until all currently queued entries have been processed.\n * This is only used in tests.\n /\n async allProcessed() {\n if (this.#running) {\n const {promise, resolve} = resolver();\n this.#queue.enqueue(['ready', resolve]);\n await promise;\n }\n }\n\n /\n Stops the storer and waits up to a drain timeout for entries to drain,\n * throwing an exception if it doesn't drain in time, in order to abort\n * the server when PG (or the connection to it) appears to be wedged.\n /\n async stop() {\n if (this.#running) {\n this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n this.#queue.enqueue('stop');\n }\n if (\n (await orTimeout(this.#stopped, this.#drainTimeoutMs)) === 'timed-out'\n ) {\n throw new AbortError(\n `changeLog did not drain within ${this.#drainTimeoutMs}ms`,\n );\n }\n }\n}\n\n/\n Extracts the stringified change message from the stringified\n * stream message (e.g. the second tuple element). This optimization\n * facilitates stringifying (and sharing the result of) the stream\n * message exactly once, but storing only the change message substring\n * in the changeLog for backwards compatibility.\n */\nexport function extractChangeSubstring(\n streamMessageJSON: string,\n tag: Change['tag'] \| undefined,\n) {\n switch (tag) {\n case 'begin':\n case 'commit':\n // e.g.\n // [\"begin\",<message-json>,{\"commitWatermark\":\"92fj2d0s\"}]\n // [\"commit\",<message-json>,{\"watermark\":\"92fj2d0s\"}]\n return streamMessageJSON.substring(\n streamMessageJSON.indexOf(',') + 1,\n streamMessageJSON.lastIndexOf(','),\n );\n default:\n // [\"data\",<message-json>]\n return streamMessageJSON.substring(\n streamMessageJSON.indexOf(',') + 1,\n streamMessageJSON.lastIndexOf(']'),\n );\n }\n}\n\ntype ChangeLogEntry = {\n watermark: string;\n tag: string;\n change: string;\n};\n\nfunction toDownstream(entry: ChangeLogEntry): WatermarkedChange {\n const {watermark, change} = entry;\n const tag = entry.tag as ChangeTag;\n switch (tag) {\n case 'begin':\n return [\n watermark,\n tag,\n `[\"begin\",${change},{\"commitWatermark\":\"${watermark}\"}]`,\n ];\n case 'commit':\n return [\n watermark,\n tag,\n `[\"commit\",${change},{\"watermark\":\"${watermark}\"}]`,\n ];\n case 'rollback':\n return [watermark, tag, `[\"rollback\",${change}]`];\n default:\n return [watermark, tag, `[\"data\",${change}]`];\n }\n}\n\nexport class PurgeLock {\n readonly #lc: LogContext;\n readonly #tx: TransactionPool;\n readonly replicaVersion: string;\n readonly minWatermark: string;\n\n constructor(\n lc: LogContext,\n tx: TransactionPool,\n replicaVersion: string,\n watermark: string,\n ) {\n this.#lc = lc;\n this.#tx = tx;\n this.replicaVersion = replicaVersion;\n this.minWatermark = watermark;\n }\n\n #released = false;\n\n async release() {\n if (this.#released) {\n return;\n }\n this.#released = true;\n this.#tx.setDone();\n await this.#tx\n .done()\n .catch(e => this.#lc.warn?.(`error from purge-lock release`, e));\n this.#lc.info?.(`released purge lock on ${this.minWatermark}`);\n }\n}\n\nexport class PurgeLocker {\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #db: PostgresDB;\n\n constructor(lc: LogContext, shard: ShardID, db: PostgresDB) {\n this.#lc = lc.withContext('component', 'purge-locker');\n this.#shard = shard;\n this.#db = db;\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n async acquire() {\n const tx = new TransactionPool(this.#lc, {mode: Mode.READ_COMMITTED}).run(\n this.#db,\n );\n const row = await tx.processReadTask(\n sql => sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR SHARE \n `,\n );\n if (row.length === 0) {\n this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);\n tx.setDone();\n await tx.done();\n return null;\n }\n const [{watermark}] = row;\n const [{replicaVersion}] = await tx.processReadTask(\n sql => sql<{replicaVersion: string}[]>`\n SELECT \"replicaVersion\" FROM ${this.#cdc('replicationConfig')}\n `,\n );\n this.#lc.info?.(\n `locked watermark ${watermark} from being purged from replica@${replicaVersion}`,\n );\n return new PurgeLock(this.#lc, tx, replicaVersion, watermark);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAmFA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkD5D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,MAAkB;CACxC;CACA;CACA;CACA;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA,EACE,iCACA,oBACA,oBACA,iBAAiB,OAEnB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKC,SAAS;EACd,KAAKC,UAAU;EACf,KAAKC,oBAAoB;EACzB,KAAKC,qBAAqB;EAC1B,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,cAAc;EACnB,KAAKC,WAAW;EAChB,KAAKG,sBAAsB;EAC3B,KAAKC,sBAAsB,KAAK,IAAI,GAAG,kBAAkB;EACzD,KAAKC,kBAAkB;EAEvB,MAAM,YAAY,kBAAkB;EACpC,KAAKH,+BACF,UAAU,kBAAkB,UAAU,kBACvC;EAEF,KAAKV,IAAI,OACP,gBAAgB,KAAKU,8BAA8B,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAE/E,EAAC,UAAS,CACZ;CACF;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKL,IAAI,GAAG,UAAU,KAAKJ,MAAM,EAAE,GAAG,OAAO;CACtD;CAEA,MAAM,gBAAgB,WAA8B;EAClD,MAAM,KAAK,KAAKI;EAChB,MAAM,QAAQ,KAAKH;EACnB,MAAM,eAAe,KAAKC;EAC1B,MAAM,gBAAgB,KAAKC;EAE3B,MAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,cAAc,KAAK;EAC5B,KAAKJ,IAAI,OAAO,yBAAyB,qBAAqB;EAC9D,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,EAAE,UAAU,KAAKc,KAAK,kBAAkB,EAAE,OAAO,GAAG;GAAC;GAAO,cAAc;EAAmB,CAAC;EACpG,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,KAAKd,IAAI,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,KAC1D;EAEA,IAAI,WAKF,UAAe,QAAQ;CAE3B;CAEA,MAAM,yCAGH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,UAAU,MAAM,MACxC,KAAKK,MACL,QAAO,CACL,GAA8B;sCACA,KAAKS,KAAK,kBAAkB,KAK1D,GAAG;;;;;;;;;iBASM,KAAKA,KAAK,aAAa,EAAE;sBACpB,KAAKA,KAAK,eAAe,EAAE;;;SAI3C,GACA,EAAC,MAAM,SAAa,CACtB;EAEA,OAAO;GACL;GACA,kBAAkB,MAAQ,QAAQ,sBAAsB;EAC1D;CACF;CAEA,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,KAAKT,GAC5B;qDACyC,KAAKS,KAAK,WAAW;EACtE,OAAO;CACT;CAEA,mBAAmB,WAAoC;EACrD,OAAO,MAAM,KAAKT,KAAK,OAAM,QAAO;GAGlC,MAAM,GAA0B;kCACJ,KAAKS,KAAK,WAAW,EAAE;;;;GASnD,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;;;;;;;;iBASvC,KAAKA,KAAK,WAAW,EAAE;;wBAEhB,KAAKA,KAAK,WAAW,EAAE,qBAAqB,UAAU;;;;GAKxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;8BACvB,KAAKA,KAAK,kBAAkB,EAAE;GACtD,IAAI,UAAU,KAAKZ,SACjB,MAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,OAClF;GAEF,OAAO,OAAO,OAAO;EACvB,CAAC;CACH;;;;CAKA,MAAM,WAAmB,MAAwB;EAI/C,MAAM,OAAO,WAAW,UAAU,IAAI;EACtC,KAAKa,2BAA2B,KAAK;EAErC,MAAM,SAAS,KAAK;EACpB,KAAKN,OAAO,QAAQ;GAClB;GACA;GACA;GACA,aAAa,MAAM,IAAI,OAAO;EAChC,CAAC;EAED,OAAO;CACT;CAEA,QAAQ;EACN,KAAKA,OAAO,QAAQ,CAAC,OAAO,CAAC;CAC/B;CAEA,OAAO,GAA4B;EACjC,KAAKA,OAAO,QAAQ,CAAC;CACvB;CAEA,QAAQ,YAAwB,MAAsB;EACpD,KAAKA,OAAO,QAAQ,CAAC,cAAc;GAAC;GAAY;EAAI,CAAC,CAAC;CACxD;CAEA,gBAAuC;CACvC;CAEA,eAA0C;EACxC,IAAI,CAAC,KAAKO,UACR;EAEF,IACE,KAAKC,kBAAkB,QACvB,KAAKF,0BAA0B,KAAKL,6BACpC;GACA,KAAKV,IAAI,OACP,+BAA+B,KAAKS,OAAO,KAAK,EAAE,qBAAqB,KAAKM,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,2SAW9H;GACA,KAAKE,gBAAgB,SAAS;GAC9B,KAAKC,0BAA0B,IAAI;EACrC;EACA,OAAO,KAAKD,eAAe;CAC7B;CAEA,0BAA0B,YAAqB;EAC7C,aAAa,KAAKE,oBAAoB;EACtC,KAAKA,uBAAuB,aACxB,iBAAiB;GACf,KAAKF,eAAe,OAClB,IAAI,WACF,0CAA0C,KAAKN,oBAAoB,6CACrE,CACF;GACA,KAAKM,gBAAgB;EACvB,GAAG,KAAKN,mBAAmB,IAC3B,KAAA;CACN;CAEA,4BAA4B;EAC1B,IAAI,KAAKM,kBAAkB,MAEzB,IACE,KAAKF,0BACL,KAAKL,8BAA8B,IACnC;GACA,KAAKV,IAAI,OACP,gCAAgC,KAAKS,OAAO,KAAK,EAAE,qBAAqB,KAAKM,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,KAC/H;GACA,KAAKE,cAAc,QAAQ;GAC3B,KAAKA,gBAAgB;GACrB,KAAKC,0BAA0B,KAAK;EACtC,OACE,KAAKA,0BAA0B,IAAI;CAGzC;;;;;;;;;;;;CAaA,gBAAgB,IAAsD;EACpE,MAAM,EAAC,UAAS;EAChB,IAAI,MAAM,WAAW,GACnB,OAAO,GAAG;EAEZ,GAAG,QAAQ,CAAC;EACZ,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG;sBACa,KAAKJ,KAAK,WAAW,EAAE;;mCAEV,MAAM;;;;;YAMrC,CAAC;EACD,OAAO,GAAG;CACZ;CAEA,WAAW;;;;;CAMX,MAAM,MAAM;EACV,OAAO,CAAC,KAAKE,UAAU,2BAA2B;EAElD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,SAAS;EAC5D,KAAKA,WAAW;EAChB,KAAKI,WAAW;EAEhB,KAAKpB,IAAI,OAAO,iBAAiB;EACjC,IAAI;EACJ,IAAI;GACF,MAAM,KAAKqB,cAAc;EAC3B,SAAS,GAAG;GACV,MAAM;GACN,MAAM;EACR,UAAU;GAER,IAAI,KAAKJ,kBAAkB,MAAM;IAC/B,KAAKA,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;GACA,KAAKC,0BAA0B,KAAK;GACpC,KAAKI,oBACH,KAAKb,OAAO,MAAM,EAAE,QAAO,UAAS,UAAU,KAAA,CAAS,GACvD,GACF;GACA,KAAKO,WAAW;GAChB,cAAc;GACd,KAAKhB,IAAI,OAAO,gBAAgB;EAClC;CACF;CAEA,oBAAoB,OAAqB,GAAY;EACnD,IAAI,MAAM,WAAW,GACnB;EAEF,KAAKA,IAAI,OACP,aAAa,MAAM,OAAO,kCAC5B;EACA,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,sBAAsB;EAC1E,KAAK,MAAM,SAAS,OAAO;GACzB,IAAI,UAAU,QACZ;GAGF,QADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;KAC3B,KAAKA,IAAI,OAAO,iBAAiB,WAAW,IAAI;KAChD,WAAW,KAAK,GAAG;KACnB;IACF;GACF;EACF;CACF;CAEA,MAAMqB,gBAAgB;EACpB,IAAI,KAAgC;EACpC,IAAI;EAEJ,MAAM,eAAoC,CAAC;EAC3C,IAAI;GACF,QAAQ,MAAM,MAAM,KAAKZ,OAAO,QAAQ,OAAO,QAAQ;IACrD,MAAM,CAAC,WAAW;IAClB,QAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;MACxB,YAAY;MACZ;KACF;KACA,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;MACvB,IAAI,IACF,aAAa,KAAK,UAAU;WAE5B,MAAM,KAAKc,cAAc,CAAC,UAAU,CAAC;MAEvC;KACF;KACA,KAAK;MACH,KAAKhB,YAAY,GAAG;MACpB;KACF,KAAK;MACH,IAAI,IAAI;OACN,GAAG,KAAK,MAAM;OACd,MAAM,GAAG,KAAK,KAAK;OACnB,KAAK;MACP;MACA;IAEJ;IAEA,MAAM,CAAC,GAAG,WAAW,MAAM,UAAU;IACrC,MAAM,MAAM,QAAQ;IACpB,KAAKQ,2BAA2B,KAAK;IAErC,IAAI,QAAQ,SAAS;KACnB,OAAO,CAAC,IAAI,+CAA+C;KAC3D,MAAM,EAAC,SAAS,SAAS,WAAU,SAA2B;KAC9D,QAAa,YAAY,CAAC,CAAC;KAC3B,KAAK;MACH,MAAM,IAAI,gBACR,KAAKf,IAAI,YAAY,aAAa,SAAS,GAC3C;OACE,MAAM;OACN,0BAA0B,KAAKW;MACjC,CACF;MACA,oBAAoB;MACpB,KAAK;MACL,0BAA0B;MAC1B,KAAK,CAAC,OAAO;MACb,OAAO,CAAC;MACR,WAAW,KAAA;KACb;KACA,GAAG,KAAK,IAAI,KAAKN,GAAG;KAGpB,GAAQ,KAAK,SAAQ,OAAM;MACzB,EAA+B;gCACX,KAAKS,KAAK,kBAAkB,EAAE,aAAa,MAC5D,CAAC,YAAY,QAAQ,MAAM,GAC5B,MACF;MACA,OAAO,CAAC;KACV,CAAC;IACH,OAAO;KACL,OAAO,UAAU,2CAA2C,MAAM;KAClE,GAAG;IACL;IAEA,MAAM,QAAsB;KAC1B,WAAW,QAAQ,WAAW,YAAY,GAAG;KAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;KACtD,KAAK,GAAG;KAGR,QAAQ,uBAAuB,MAAM,GAAG;IAC1C;IAEA,IAAI,WAAW,QAAQ,eAAe,MAAM,GAAG;KAM7C,MAAM,KAAKU,gBAAgB,EAAE;KAC7B,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG,eAAe,KAAKV,KAAK,WAAW,EAAE,GAAG,IAAI,KAAK,KACrD,GAAG,KAAKW,uBAAuB,KAAK,MAAM,CAC5C,CAAC;IACH,OAAO;KAML,GAAG,MAAM,KAAK,KAAK;KACnB,IAAI,GAAG,MAAM,UAAU,KAAKb,qBAAqB;MAK/C,MAAM,YAAY,GAAG;MACrB,KAAUY,gBAAgB,EAAE;MAC5B,MAAM;KACR;IACF;IACA,KAAKE,0BAA0B;IAE/B,IAAI,QAAQ,UAAU;KAIpB,KAAUF,gBAAgB,EAAE;KAE5B,MAAM,EAAC,UAAS,MAAM,GAAG;KACzB,IAAI,UAAU,KAAKtB,SAEjB,GAAG,KAAK,KACN,IAAI,WACF,2CAA2C,OAC7C,CACF;UACK;MAEL,MAAM,gBAAgB;MACtB,GAAQ,KAAK,SAAQ,OAAM,CACzB,EAAE;qBACK,KAAKY,KAAK,kBAAkB,EAAE,OAAO,GAAG,EAAC,cAAa,CAAC,GAChE,CAAC;MACD,GAAG,KAAK,QAAQ;KAClB;KAEA,MAAM,GAAG,KAAK,KAAK;KAGnB,IAAI,GAAG,KACL,KAAKP,YAAY;MAAC;MAAU;MAAQ,EAAC,UAAS;KAAC,CAAC;KAElD,KAAK;KAIL,MAAM,KAAKgB,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,QAAQ,YAAY;KAG7B,GAAG,KAAK,MAAM;KACd,MAAM,GAAG,KAAK,KAAK;KACnB,KAAK;KAEL,MAAM,KAAKA,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD;GACF;EACF,SAAS,GAAG;GACV,aAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GACzD,MAAM;EACR;CACF;CAEA,MAAMA,cAAc,MAA2B;EAC7C,IAAI,KAAK,WAAW,GAClB;EAGF,MAAM,SAAS,IAAI,gBACjB,KAAKvB,IAAI,YAAY,QAAQ,SAAS,GACtC,EAAC,MAAM,SAAa,CACtB;EACA,OAAO,IAAI,KAAKK,GAAG;EAEnB,IAAI;EACJ,IAAI;GAKF,CAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAA8B;sCACP,KAAKS,KAAK,kBAAkB,EAAE;OAE9D;EACF,SAAS,GAAG;GACV,KAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GAC7C,MAAM;EACR;EAIA,QAAa,IACX,KAAK,KAAI,QAAO,KAAKa,SAAS,KAAK,eAAe,MAAM,CAAC,CAC3D,EAAE,cAAc,OAAO,QAAQ,CAAC;CAClC;CAEA,MAAMA,SACJ,EAAC,YAAY,KAAK,QAClB,eACA,QACA;EACA,IAAI;GACF,MAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,MAAM,QAAQ,KAAK,IAAI;IAIvB,IAAI,iBAAiB,IAAI,cAAc,KAAKrB;IAC5C,IAAI,QAAQ;IACZ,IAAI;IAEJ,WAAW,MAAM,WAAW,EAA6B;sEACK,KAAKQ,KAAK,WAAW,EAAE;gCAC7D,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,GAAI,GAAG;KAOxC,MAAM,QAAQ,YAAY,IAAI;KAC9B,MAAM;KACN,MAAM,UAAU,YAAY,IAAI,IAAI;KACpC,IAAI,mBACF,KAAKd,IAAI,UAAU,MAAM,SAAS,WAChC,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,IAAI,GAAG,0CAChD;KAGF,KAAK,MAAM,SAAS,SAClB,IAAI,MAAM,cAAc,IAAI,WAG1B,iBAAiB;UACZ,IAAI,gBAAgB;MACzB,oBAAoB,IAAI,QAAQ,aAAa,KAAK,CAAC;MACnD;KACF,OAAO,IAAI,SAAS,UAClB,MAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,EACvF;UACK;MACL,KAAKA,IAAI,OACP,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,EAC7F;MACA,IAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,EACjF;MACA;KACF;IAEJ;IACA,IAAI,gBAAgB;KAClB,MAAM;KACN,KAAKA,IAAI,OACP,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,IAAI,IAAI,MACd,KACH;IACF,OAWE,KAAKA,IAAI,OACP,cAAc,IAAI,GAAG,gBAAgB,IAAI,UAAU,4CACjB,cAAc,wCAElD;IAMF,IAAS,YAAY;GACvB,CAAC;EACH,SAAS,KAAK;GACZ,KAAKA,IAAI,QAAQ,sCAAsC,IAAI,MAAM,GAAG;GACpE,IAAI,eAAe,iBAAiB;IAClC,MAAM,kBAAkB,KAAKK,KAAK,KAAKJ,MAAM;IAC7C,KAAKO,SAAS,GAAG;GACnB;GACA,IAAI,KAAK,GAAG;EACd;CACF;;;;;CAMA,uBAAuB,KAA0B,QAAsB;EACrE,MAAM,QAA+B,CAAC;EAEtC,QAAQ,OAAO,KAAf;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,KAAK,KAAKoB,yBAAyB,KAAK,OAAO,QAAQ,CAAC;IAC9D;GACF;GAEA,KAAK,gBAAgB;IACnB,MAAM,EAAC,MAAM,UAAU,aAAY;IACnC,IAAI,UACF,MAAM,KAAK,KAAKA,yBAAyB,KAAK,MAAM,QAAQ,CAAC;IAE/D,IAAI,UACF,OAAO,QAAQ,QAAQ,EAAE,SAAS,CAAC,KAAK,cAAc;KACpD,MAAM,KACJ,KAAKC,0BAA0B,KAAK,MAAM,KAAK,QAAQ,CACzD;IACF,CAAC;IAEH;GACF;GAEA,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,OAAO,IAAI;KAAQ,OAAO,OAAO,IAAI;IAAI;IAC9D,MAAM,KACJ,GAAG,UAAU,KAAKf,KAAK,eAAe,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,MAC3D;IACA;GACF;GAEA,KAAK,cAAc;IACjB,MAAM,EACJ,IAAI,EAAC,QAAQ,WACX;IACJ,MAAM,KACJ,GAAG,eAAe,KAAKA,KAAK,eAAe,EAAE;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MACnD;IACA;GACF;GAEA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;IACjD,IAAI,eACF,MAAM,KAAK,KAAKc,yBAAyB,KAAK,OAAO,aAAa,CAAC;IAErE,IAAI,UACF,MAAM,KACJ,KAAKC,0BAA0B,KAAK,OAAO,OAAO,MAAM,QAAQ,CAClE;IAEF;GACF;GAEA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OAAO,EAAC,QAAQ,MAAM,SACtB,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SACd,MAAM,KACJ,GAAG,UAAU,KAAKf,KAAK,aAAa,EAAE,kBAAkB,QAAQ;mCACzC,OAAO,iBAAiB,MAAM,kBAAkB,SACzE;IAEF;GACF;GAEA,KAAK,eAAe;IAClB,MAAM,EACJ,OAAO,EAAC,QAAQ,QAChB,WACE;IACJ,MAAM,KACJ,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,QAC1E;IACA;GACF;GAEA,KAAK,sBAAsB;IACzB,MAAM,EACJ,UAAU,EAAC,QAAQ,MAAM,OAAO,UAChC,YACE;IACJ,MAAM,OAAO,CAAC,GAAG,OAAO,SAAS,GAAG,OAAO;IAC3C,MAAM,KACJ,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,IAAI,GACpF;GACF;EACF;EACA,OAAO;CACT;CAEA,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;EAAQ;EACtD,OAAO,GAAG;sBACQ,KAAKA,KAAK,eAAe,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEnC,IAAI,GAAG,EAAE;;CAEjC;CAEA,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;EAAQ;EAC/D,OAAO,GAAG;sBACQ,KAAKA,KAAK,aAAa,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEjC,IAAI,GAAG,EAAE;;CAEjC;;;;;CAMA,MAAM,eAAe;EACnB,IAAI,KAAKE,UAAU;GACjB,MAAM,EAAC,SAAS,YAAW,SAAS;GACpC,KAAKP,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;GACtC,MAAM;EACR;CACF;;;;;;CAOA,MAAM,OAAO;EACX,IAAI,KAAKO,UAAU;GACjB,KAAKhB,IAAI,OAAO,YAAY,KAAKS,OAAO,KAAK,EAAE,mBAAmB;GAClE,KAAKA,OAAO,QAAQ,MAAM;EAC5B;EACA,IACG,MAAM,UAAU,KAAKW,UAAU,KAAKP,eAAe,MAAO,aAE3D,MAAM,IAAI,WACR,kCAAkC,KAAKA,gBAAgB,GACzD;CAEJ;AACF;;;;;;;;AASA,SAAgB,uBACd,mBACA,KACA;CACA,QAAQ,KAAR;EACE,KAAK;EACL,KAAK,UAIH,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;EACF,SAEE,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;CACJ;AACF;AAQA,SAAS,aAAa,OAA0C;CAC9D,MAAM,EAAC,WAAW,WAAU;CAC5B,MAAM,MAAM,MAAM;CAClB,QAAQ,KAAR;EACE,KAAK,SACH,OAAO;GACL;GACA;GACA,YAAY,OAAO,uBAAuB,UAAU;EACtD;EACF,KAAK,UACH,OAAO;GACL;GACA;GACA,aAAa,OAAO,iBAAiB,UAAU;EACjD;EACF,KAAK,YACH,OAAO;GAAC;GAAW;GAAK,eAAe,OAAO;EAAE;EAClD,SACE,OAAO;GAAC;GAAW;GAAK,WAAW,OAAO;EAAE;CAChD;AACF;AAEA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,gBACA,WACA;EACA,KAAKb,MAAM;EACX,KAAK8B,MAAM;EACX,KAAK,iBAAiB;EACtB,KAAK,eAAe;CACtB;CAEA,YAAY;CAEZ,MAAM,UAAU;EACd,IAAI,KAAKC,WACP;EAEF,KAAKA,YAAY;EACjB,KAAKD,IAAI,QAAQ;EACjB,MAAM,KAAKA,IACR,KAAK,EACL,OAAM,MAAK,KAAK9B,IAAI,OAAO,iCAAiC,CAAC,CAAC;EACjE,KAAKA,IAAI,OAAO,0BAA0B,KAAK,cAAc;CAC/D;AACF;AAEA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB,IAAgB;EAC1D,KAAKA,MAAM,GAAG,YAAY,aAAa,cAAc;EACrD,KAAKC,SAAS;EACd,KAAKI,MAAM;CACb;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKA,IAAI,GAAG,UAAU,KAAKJ,MAAM,EAAE,GAAG,OAAO;CACtD;CAEA,MAAM,UAAU;EACd,MAAM,KAAK,IAAI,gBAAgB,KAAKD,KAAK,EAAC,MAAM,eAAmB,CAAC,EAAE,IACpE,KAAKK,GACP;EACA,MAAM,MAAM,MAAM,GAAG,iBACnB,QAAO,GAA0B;8BACT,KAAKS,KAAK,WAAW,EAAE;;;KAIjD;EACA,IAAI,IAAI,WAAW,GAAG;GACpB,KAAKd,IAAI,OAAO,4CAA4C;GAC5D,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,MAAM,CAAC,EAAC,eAAc;EACtB,MAAM,CAAC,EAAC,oBAAmB,MAAM,GAAG,iBAClC,QAAO,GAA+B;uCACL,KAAKc,KAAK,mBAAmB,EAAE;OAElE;EACA,KAAKd,IAAI,OACP,oBAAoB,UAAU,kCAAkC,gBAClE;EACA,OAAO,IAAI,UAAU,KAAKA,KAAK,IAAI,gBAAgB,SAAS;CAC9D;AACF"}

package/out/zero-cache/src/services/litestream/commands.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAM3D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;~~AAkC5D~~,KAAK,kBAAkB,GAAG;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAUF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;gBAErC,SAAS,EAAE,MAAM,GAAG,SAAS;CAG1C;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,iBA+~~C9C~~;~~AAiPD~~,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA6DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}
1	+ {"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAM3D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAsC5D,KAAK,kBAAkB,GAAG;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAUF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;gBAErC,SAAS,EAAE,MAAM,GAAG,SAAS;CAG1C;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,iBA+D9C;AAsPD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA6DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}

package/out/zero-cache/src/services/litestream/commands.js CHANGED Viewed

@@ -14,6 +14,8 @@ import { spawn } from "node:child_process";
 import { existsSync, statSync } from "node:fs";
 //#region ../zero-cache/src/services/litestream/commands.ts
 var RETRY_INTERVAL_MS = 3e3;
+var MAX_RESTORE_ATTEMPTS = 3;
+var RESTORE_RETRY_DELAY_MS = 5e3;
 var BackupNotFoundException = class extends Error {
 	static name = "BackupNotFoundException";
 	constructor(backupURL) {
@@ -32,8 +34,16 @@ async function restoreReplica(lc, config, replicaConstraints) {
 	let backupURL = config.litestream.backupURL;
 	let result;
 	try {
-		for (;;) {
-			const attempt = await tryRestore(lc, config, replicaConstraints, role);
+		for (let attemptNum = 1;; attemptNum++) {
+			let attempt;
+			try {
+				attempt = await tryRestore(lc, config, replicaConstraints, role);
+			} catch (e) {
+				if (!replicaConstraints || attemptNum === MAX_RESTORE_ATTEMPTS) throw e;
+				lc.warn?.(`litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`, e);
+				await sleep(RESTORE_RETRY_DELAY_MS);
+				continue;
+			}
 			backupURL = attempt.backupURL;
 			if (attempt.restored) {
 				result = attempt.result;
@@ -146,7 +156,8 @@ async function tryRestore(lc, config, replicaConstraints, role) {
 				...attrs,
 				result: "error"
 			});
-			throw e;
+			const output = [stdout, stderr].map((value) => value.trim()).filter(Boolean).join("\n");
+			throw new Error(output ? `${String(e)}\n${output}` : String(e), { cause: e });
 		}
 		if (!existsSync(config.replica.file)) {
 			result = "no_backup";

package/out/zero-cache/src/services/litestream/commands.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, statSync} from 'node:fs';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n SnapshotMessage,\n SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreRuns,\n litestreamRestoreValidationDuration,\n litestreamRestoreWaitDuration,\n type LitestreamRole,\n} from './metrics.ts';\n\nconst RETRY_INTERVAL_MS = 3000;\n\ntype ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\ntype RestoreResult = 'success' \| 'no_backup' \| 'invalid_replica' \| 'error';\n\ntype RestoreAttempt = {\n restored: boolean;\n backupURL: string \| undefined;\n result: RestoreResult;\n};\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string \| undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\n/*\n @param replicaConstraints The constraints of the restored backup when\n * restoring for the change-streamer (replication-manager). For the\n * view-syncer, this should be unspecified so that the constraints are\n * retrieved from the replication-manager via the snapshot protocol.\n /\nexport async function restoreReplica(\n lc: LogContext,\n config: ZeroConfig,\n replicaConstraints: ReplicaConstraints \| null,\n) {\n const start = performance.now();\n const role: LitestreamRole = replicaConstraints\n ? 'replication_manager'\n : 'view_syncer';\n let backupURL = config.litestream.backupURL;\n let result: RestoreResult \| undefined;\n // View-syncers (no replicaConstraints) wait indefinitely for the\n // replication-manager to publish a restorable backup. On a fresh stack the\n // first backup is not durable until the initial sync completes and litestream\n // uploads the initial snapshot, which can take many minutes for a large\n // replica. The platform's startup probe budget (which scales with replica\n // size) is the backstop, so restoreReplica must not impose its own shorter\n // cap and self-terminate while the backup is still being produced.\n try {\n for (;;) {\n const attempt = await tryRestore(lc, config, replicaConstraints, role);\n backupURL = attempt.backupURL;\n if (attempt.restored) {\n result = attempt.result;\n return;\n }\n if (replicaConstraints) {\n // The replication-manager restores against explicit constraints, so a\n // missing backup is fatal (e.g. the litestream URL was purposefully\n // changed to force a resync). Only the view-syncer (no constraints)\n // waits for a backup to appear.\n result = attempt.result;\n throw new BackupNotFoundException(config.litestream.backupURL);\n }\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n } catch (e) {\n if (e instanceof BackupNotFoundException && result === undefined) {\n result = 'no_backup';\n }\n throw e;\n } finally {\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n\nfunction getLitestream(\n mode: 'restore' \| 'replicate',\n config: ZeroConfig,\n logLevelOverride?: LogLevel,\n backupURLOverride?: string,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupURL,\n logLevel,\n configPath,\n endpoint,\n region,\n port = config.port + 2,\n checkpointThresholdMB,\n minCheckpointPageCount = checkpointThresholdMB 250, // SQLite page size is 4KB\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n incrementalBackupIntervalMinutes,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n\n const litestream =\n // The v0.5.8+ litestream executable can restore from either the new LTX\n // format or the legacy WAL format, allowing forwards-compatibility /\n // rollback safety with zero-cache versions that backup to LTX.\n (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: config.replica.file,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours,\n ),\n ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: config.log.format,\n ['LITESTREAM_CONFIG']: configPath,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n },\n };\n}\n\nasync function tryRestore(\n lc: LogContext,\n config: ZeroConfig,\n replicaConstraints: ReplicaConstraints \| null,\n role: LitestreamRole,\n): Promise<RestoreAttempt> {\n let snapshotStatus: SnapshotStatus \| undefined;\n if (!replicaConstraints) {\n // view-syncers fetch replica constraints from the replication-manager\n // via the snapshot protocol.\n const waitStart = performance.now();\n snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n litestreamRestoreWaitDuration().recordMs(\n performance.now() - waitStart,\n litestreamRestoreMetricAttrs(config, role, snapshotStatus.backupURL),\n );\n lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n replicaConstraints = snapshotStatus;\n }\n\n const backupURL = snapshotStatus?.backupURL ?? config.litestream.backupURL;\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n let result: RestoreResult = 'error';\n try {\n const replicaExistedBeforeRestore = existsSync(config.replica.file);\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n snapshotStatus?.backupURL,\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided whether the failure is retriable. The captured output is\n // re-surfaced through `lc` below with a distinct message per attempt, so a\n // retriable failure is routed to a non-paging warning and only a post-retry\n // failure pages. See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n config.replica.file,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.stderr.on('data', chunk => (stderr += chunk));\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n // A successful restore does not emit ERROR-level output; forward\n // litestream's own (minimal, debug-level) restore logging to the pod's\n // stdout/stderr, matching the previous `stdio: 'inherit'` behavior.\n if (stdout) {\n process.stdout.write(stdout);\n }\n if (stderr) {\n process.stderr.write(stderr);\n }\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n throw e;\n }\n if (!existsSync(config.replica.file)) {\n result = 'no_backup';\n return {restored: false, backupURL, result};\n }\n const validationStart = performance.now();\n const valid = replicaIsValid(lc, config.replica.file, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: valid ? 'success' : 'invalid_replica'},\n );\n if (!valid) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(config.replica.file);\n return {restored: false, backupURL, result};\n }\n result = 'success';\n if (!replicaExistedBeforeRestore) {\n litestreamRestoredDbBytes().add(statSync(config.replica.file).size, {\n ...attrs,\n result: 'success',\n });\n }\n return {restored: true, backupURL, result};\n } finally {\n litestreamRestoreAttempts().add(1, {\n ...attrs,\n result,\n });\n }\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints,\n) {\n let db: Database \| undefined;\n try {\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return false;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n return true;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: ZeroConfig,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' \| 'error' \| 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/*\n Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n \n This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * believes an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n \n Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n /\nexport async function getLastBackupTime(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, 'snapshots'),\n listBackupCreatedTimes(lc, config, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${config.litestream.backupURL}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/\n Runs `litestream <snapshots\|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n \n ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n /\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: ZeroConfig,\n command: 'snapshots' \| 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' \| 'empty' \| 'timeout' \| 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config);\n const proc = spawn(litestream, [command, config.replica.file], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/\n Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n \n Exported for testing.\n /\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' \| 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 \|\|\n created === undefined \|\|\n created === 'created' / header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n\nfunction reserveAndGetSnapshotStatus(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<SnapshotStatus> {\n const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n void (async function () {\n const abort = new AbortController();\n process.on('SIGINT', () => abort.abort());\n process.on('SIGTERM', () => abort.abort());\n\n for (let i = 0; ; i++) {\n let err: unknown;\n try {\n let resolved = false;\n const stream = await reserveSnapshot(lc, config);\n for await (const msg of stream) {\n // Capture the value of the status message that the change-streamer\n // backup monitor returns, and hold the connection open to\n // \"reserve\" the snapshot and prevent change log cleanup.\n resolve(msg[1]);\n resolved = true;\n }\n // The change-streamer itself closes the connection when the\n // subscription is started (or the reservation retried).\n if (resolved) {\n break;\n }\n } catch (e) {\n err = e;\n }\n // Retry in the view-syncer since it cannot proceed until it connects\n // to a (compatible) replication-manager. In particular, a\n // replication-manager that does not support the view-syncer's\n // change-streamer protocol will close the stream with an error; this\n // retry logic essentially delays the startup of a view-syncer until\n // a compatible replication-manager has been rolled out, allowing\n // replication-manager and view-syncer services to be updated in\n // parallel.\n lc.warn?.(\n `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n String(err),\n );\n try {\n await sleep(5000, abort.signal);\n } catch (e) {\n return reject(e);\n }\n }\n })();\n\n return status;\n}\n\nfunction reserveSnapshot(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n assertNormalized(config);\n const {taskID, change, changeStreamer} = config;\n const shardID = getShardConfig(config);\n\n const changeStreamerClient = new ChangeStreamerHttpClient(\n lc,\n shardID,\n change.db,\n changeStreamer.uri,\n );\n\n return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAyCA,IAAM,oBAAoB;AAe1B,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;EACzC,MAAM,uBAAuB,WAAW;CAC1C;AACF;;;;;;;AAQA,eAAsB,eACpB,IACA,QACA,oBACA;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAuB,qBACzB,wBACA;CACJ,IAAI,YAAY,OAAO,WAAW;CAClC,IAAI;CAQJ,IAAI;EACF,SAAS;GACP,MAAM,UAAU,MAAM,WAAW,IAAI,QAAQ,oBAAoB,IAAI;GACrE,YAAY,QAAQ;GACpB,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,IAAI,oBAAoB;IAKtB,SAAS,QAAQ;IACjB,MAAM,IAAI,wBAAwB,OAAO,WAAW,SAAS;GAC/D;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,SAAS,GAAG;EACV,IAAI,aAAa,2BAA2B,WAAW,KAAA,GACrD,SAAS;EAEX,MAAM;CACR,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,MAAM,SACtC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAEA,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,QACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAQX,OAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,iCAAiC;EAGlD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,SAAS;IAClE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAe,WACb,IACA,QACA,oBACA,MACyB;CACzB,IAAI;CACJ,IAAI,CAAC,oBAAoB;EAGvB,MAAM,YAAY,YAAY,IAAI;EAClC,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;EAC7D,8BAA8B,EAAE,SAC9B,YAAY,IAAI,IAAI,WACpB,6BAA6B,QAAQ,MAAM,eAAe,SAAS,CACrE;EACA,GAAG,OAAO,yBAAyB,eAAe,WAAW;EAC7D,qBAAqB;CACvB;CAEA,MAAM,YAAY,gBAAgB,aAAa,OAAO,WAAW;CACjE,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,MAAM,8BAA8B,WAAW,OAAO,QAAQ,IAAI;EAClE,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,SAClB;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE,OAAO;EACX,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAQD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,OAAO,QAAQ;EACjB,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;GAIA,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;GAE7B,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;EAE/B,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM;EACR;EACA,IAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GAAG;GACpC,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,OAAO,QAAQ,MAAM,kBAAkB;EACxE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,OAAO,QAAQ,IAAI;GAChC,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BACH,0BAA0B,EAAE,IAAI,SAAS,OAAO,QAAQ,IAAI,EAAE,MAAM;GAClE,GAAG;GACH,QAAQ;EACV,CAAC;EAEH,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,0BAA0B,EAAE,IAAI,GAAG;GACjC,GAAG;GACH;EACF,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,mBAAmB,YAAY,gBAAgB;GACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;GACA,OAAO;EACT;EACA,IAAI,YAAY,YAAY,cAAc;GACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;GACA,OAAO;EACT;EACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW,WAAW;CACxE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,WAAW,GAC9C,uBAAuB,IAAI,QAAQ,KAAK,CAC1C,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WAAW,WAC9D;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,OAAO,QAAQ,IAAI,GAAG;EAC7D;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,iBAAiB,MAAM;CACvB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}
1	+ {"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, statSync} from 'node:fs';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n SnapshotMessage,\n SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreRuns,\n litestreamRestoreValidationDuration,\n litestreamRestoreWaitDuration,\n type LitestreamRole,\n} from './metrics.ts';\n\nconst RETRY_INTERVAL_MS = 3000;\n// A short retry is much cheaper than an initial Postgres sync, while keeping a\n// persistent failure from delaying startup appreciably.\nconst MAX_RESTORE_ATTEMPTS = 3;\nconst RESTORE_RETRY_DELAY_MS = 5_000;\n\ntype ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\ntype RestoreResult = 'success' \| 'no_backup' \| 'invalid_replica' \| 'error';\n\ntype RestoreAttempt = {\n restored: boolean;\n backupURL: string \| undefined;\n result: RestoreResult;\n};\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string \| undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\n/*\n @param replicaConstraints The constraints of the restored backup when\n * restoring for the change-streamer (replication-manager). For the\n * view-syncer, this should be unspecified so that the constraints are\n * retrieved from the replication-manager via the snapshot protocol.\n /\nexport async function restoreReplica(\n lc: LogContext,\n config: ZeroConfig,\n replicaConstraints: ReplicaConstraints \| null,\n) {\n const start = performance.now();\n const role: LitestreamRole = replicaConstraints\n ? 'replication_manager'\n : 'view_syncer';\n let backupURL = config.litestream.backupURL;\n let result: RestoreResult \| undefined;\n // View-syncers (no replicaConstraints) wait indefinitely for the\n // replication-manager to publish a restorable backup. On a fresh stack the\n // first backup is not durable until the initial sync completes and litestream\n // uploads the initial snapshot, which can take many minutes for a large\n // replica. The platform's startup probe budget (which scales with replica\n // size) is the backstop, so restoreReplica must not impose its own shorter\n // cap and self-terminate while the backup is still being produced.\n try {\n for (let attemptNum = 1; ; attemptNum++) {\n let attempt: RestoreAttempt;\n try {\n attempt = await tryRestore(lc, config, replicaConstraints, role);\n } catch (e) {\n // View-syncers retain their existing startup behavior. For replication\n // managers, retry a transient restore failure twice before falling back\n // to the existing initial-sync recovery path.\n if (!replicaConstraints \|\| attemptNum === MAX_RESTORE_ATTEMPTS) {\n throw e;\n }\n lc.warn?.(\n `litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`,\n e,\n );\n await sleep(RESTORE_RETRY_DELAY_MS);\n continue;\n }\n backupURL = attempt.backupURL;\n if (attempt.restored) {\n result = attempt.result;\n return;\n }\n if (replicaConstraints) {\n // The replication-manager restores against explicit constraints, so a\n // missing backup is fatal (e.g. the litestream URL was purposefully\n // changed to force a resync). Only the view-syncer (no constraints)\n // waits for a backup to appear.\n result = attempt.result;\n throw new BackupNotFoundException(config.litestream.backupURL);\n }\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n } catch (e) {\n if (e instanceof BackupNotFoundException && result === undefined) {\n result = 'no_backup';\n }\n throw e;\n } finally {\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n\nfunction getLitestream(\n mode: 'restore' \| 'replicate',\n config: ZeroConfig,\n logLevelOverride?: LogLevel,\n backupURLOverride?: string,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupURL,\n logLevel,\n configPath,\n endpoint,\n region,\n port = config.port + 2,\n checkpointThresholdMB,\n minCheckpointPageCount = checkpointThresholdMB 250, // SQLite page size is 4KB\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n incrementalBackupIntervalMinutes,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n\n const litestream =\n // The v0.5.8+ litestream executable can restore from either the new LTX\n // format or the legacy WAL format, allowing forwards-compatibility /\n // rollback safety with zero-cache versions that backup to LTX.\n (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: config.replica.file,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours,\n ),\n ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: config.log.format,\n ['LITESTREAM_CONFIG']: configPath,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n },\n };\n}\n\nasync function tryRestore(\n lc: LogContext,\n config: ZeroConfig,\n replicaConstraints: ReplicaConstraints \| null,\n role: LitestreamRole,\n): Promise<RestoreAttempt> {\n let snapshotStatus: SnapshotStatus \| undefined;\n if (!replicaConstraints) {\n // view-syncers fetch replica constraints from the replication-manager\n // via the snapshot protocol.\n const waitStart = performance.now();\n snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n litestreamRestoreWaitDuration().recordMs(\n performance.now() - waitStart,\n litestreamRestoreMetricAttrs(config, role, snapshotStatus.backupURL),\n );\n lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n replicaConstraints = snapshotStatus;\n }\n\n const backupURL = snapshotStatus?.backupURL ?? config.litestream.backupURL;\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n let result: RestoreResult = 'error';\n try {\n const replicaExistedBeforeRestore = existsSync(config.replica.file);\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n snapshotStatus?.backupURL,\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided how to handle the failure. The captured output is\n // included in the thrown error so the caller can log the final outcome.\n // See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n config.replica.file,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.stderr.on('data', chunk => (stderr += chunk));\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n // A successful restore does not emit ERROR-level output; forward\n // litestream's own (minimal, debug-level) restore logging to the pod's\n // stdout/stderr, matching the previous `stdio: 'inherit'` behavior.\n if (stdout) {\n process.stdout.write(stdout);\n }\n if (stderr) {\n process.stderr.write(stderr);\n }\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n const output = [stdout, stderr]\n .map(value => value.trim())\n .filter(Boolean)\n .join('\\n');\n throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n cause: e,\n });\n }\n if (!existsSync(config.replica.file)) {\n result = 'no_backup';\n return {restored: false, backupURL, result};\n }\n const validationStart = performance.now();\n const valid = replicaIsValid(lc, config.replica.file, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: valid ? 'success' : 'invalid_replica'},\n );\n if (!valid) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(config.replica.file);\n return {restored: false, backupURL, result};\n }\n result = 'success';\n if (!replicaExistedBeforeRestore) {\n litestreamRestoredDbBytes().add(statSync(config.replica.file).size, {\n ...attrs,\n result: 'success',\n });\n }\n return {restored: true, backupURL, result};\n } finally {\n litestreamRestoreAttempts().add(1, {\n ...attrs,\n result,\n });\n }\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints,\n) {\n let db: Database \| undefined;\n try {\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return false;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n return true;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: ZeroConfig,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' \| 'error' \| 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/*\n Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n \n This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * believes an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n \n Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n /\nexport async function getLastBackupTime(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, 'snapshots'),\n listBackupCreatedTimes(lc, config, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${config.litestream.backupURL}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/\n Runs `litestream <snapshots\|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n \n ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n /\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: ZeroConfig,\n command: 'snapshots' \| 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' \| 'empty' \| 'timeout' \| 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config);\n const proc = spawn(litestream, [command, config.replica.file], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/\n Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n \n Exported for testing.\n /\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' \| 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 \|\|\n created === undefined \|\|\n created === 'created' / header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n\nfunction reserveAndGetSnapshotStatus(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<SnapshotStatus> {\n const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n void (async function () {\n const abort = new AbortController();\n process.on('SIGINT', () => abort.abort());\n process.on('SIGTERM', () => abort.abort());\n\n for (let i = 0; ; i++) {\n let err: unknown;\n try {\n let resolved = false;\n const stream = await reserveSnapshot(lc, config);\n for await (const msg of stream) {\n // Capture the value of the status message that the change-streamer\n // backup monitor returns, and hold the connection open to\n // \"reserve\" the snapshot and prevent change log cleanup.\n resolve(msg[1]);\n resolved = true;\n }\n // The change-streamer itself closes the connection when the\n // subscription is started (or the reservation retried).\n if (resolved) {\n break;\n }\n } catch (e) {\n err = e;\n }\n // Retry in the view-syncer since it cannot proceed until it connects\n // to a (compatible) replication-manager. In particular, a\n // replication-manager that does not support the view-syncer's\n // change-streamer protocol will close the stream with an error; this\n // retry logic essentially delays the startup of a view-syncer until\n // a compatible replication-manager has been rolled out, allowing\n // replication-manager and view-syncer services to be updated in\n // parallel.\n lc.warn?.(\n `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n String(err),\n );\n try {\n await sleep(5000, abort.signal);\n } catch (e) {\n return reject(e);\n }\n }\n })();\n\n return status;\n}\n\nfunction reserveSnapshot(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n assertNormalized(config);\n const {taskID, change, changeStreamer} = config;\n const shardID = getShardConfig(config);\n\n const changeStreamerClient = new ChangeStreamerHttpClient(\n lc,\n shardID,\n change.db,\n changeStreamer.uri,\n );\n\n return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAyCA,IAAM,oBAAoB;AAG1B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAe/B,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;EACzC,MAAM,uBAAuB,WAAW;CAC1C;AACF;;;;;;;AAQA,eAAsB,eACpB,IACA,QACA,oBACA;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAuB,qBACzB,wBACA;CACJ,IAAI,YAAY,OAAO,WAAW;CAClC,IAAI;CAQJ,IAAI;EACF,KAAK,IAAI,aAAa,IAAK,cAAc;GACvC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WAAW,IAAI,QAAQ,oBAAoB,IAAI;GACjE,SAAS,GAAG;IAIV,IAAI,CAAC,sBAAsB,eAAe,sBACxC,MAAM;IAER,GAAG,OACD,8BAA8B,WAAW,uBAAuB,uBAAuB,cAAc,aAAa,EAAE,MAAM,qBAAqB,IAC/I,CACF;IACA,MAAM,MAAM,sBAAsB;IAClC;GACF;GACA,YAAY,QAAQ;GACpB,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,IAAI,oBAAoB;IAKtB,SAAS,QAAQ;IACjB,MAAM,IAAI,wBAAwB,OAAO,WAAW,SAAS;GAC/D;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,SAAS,GAAG;EACV,IAAI,aAAa,2BAA2B,WAAW,KAAA,GACrD,SAAS;EAEX,MAAM;CACR,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,MAAM,SACtC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAEA,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,QACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAQX,OAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,iCAAiC;EAGlD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,SAAS;IAClE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAe,WACb,IACA,QACA,oBACA,MACyB;CACzB,IAAI;CACJ,IAAI,CAAC,oBAAoB;EAGvB,MAAM,YAAY,YAAY,IAAI;EAClC,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;EAC7D,8BAA8B,EAAE,SAC9B,YAAY,IAAI,IAAI,WACpB,6BAA6B,QAAQ,MAAM,eAAe,SAAS,CACrE;EACA,GAAG,OAAO,yBAAyB,eAAe,WAAW;EAC7D,qBAAqB;CACvB;CAEA,MAAM,YAAY,gBAAgB,aAAa,OAAO,WAAW;CACjE,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,MAAM,8BAA8B,WAAW,OAAO,QAAQ,IAAI;EAClE,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,SAClB;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE,OAAO;EACX,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,OAAO,QAAQ;EACjB,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;GAIA,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;GAE7B,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;EAE/B,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GAAG;GACpC,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,OAAO,QAAQ,MAAM,kBAAkB;EACxE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,OAAO,QAAQ,IAAI;GAChC,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BACH,0BAA0B,EAAE,IAAI,SAAS,OAAO,QAAQ,IAAI,EAAE,MAAM;GAClE,GAAG;GACH,QAAQ;EACV,CAAC;EAEH,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,0BAA0B,EAAE,IAAI,GAAG;GACjC,GAAG;GACH;EACF,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,mBAAmB,YAAY,gBAAgB;GACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;GACA,OAAO;EACT;EACA,IAAI,YAAY,YAAY,cAAc;GACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;GACA,OAAO;EACT;EACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW,WAAW;CACxE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,WAAW,GAC9C,uBAAuB,IAAI,QAAQ,KAAK,CAC1C,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WAAW,WAC9D;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,OAAO,QAAQ,IAAI,GAAG;EAC7D;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,iBAAiB,MAAM;CACvB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}

package/out/zero-cache/src/types/streams.d.ts CHANGED Viewed

@@ -13,6 +13,17 @@ export type Source<T> = AsyncIterable<T> & {
      * @param err Terminate the iteration by throwing the `err` instead.
      */
     cancel: (err?: Error) => void;
+    /**
+     * Resolves the result of the specified `other` Promise, or a
+     * resolved/rejected Promise when the Source is terminated or
+     * rejects.
+     *
+     * This is similar to using `Promise.race([other, sourceDone])`, but
+     * in a memory safe way, as repeatedly calling `Promise.race([ ... ])` with
+     * a long-lived Promise results in leaking memory via a growing list of
+     * `then()` callbacks (https://github.com/nodejs/node/issues/17469).
+     */
+    doneOr<R>(other: Promise<R>): Promise<void | R>;
     /**
      * The presence of a `pipeline` iterable allows the usual "consumed-on-iterate" semantics
      * to be overridden.

package/out/zero-cache/src/types/streams.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/streams.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,QAAQ,EAGR,KAAK,aAAa,EACnB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAKL,KAAK,SAAS,EACf,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAa,KAAK,SAAS,EAAC,MAAM,oCAAoC,CAAC;AAE9E,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,EAAC,YAAY,EAAE,KAAK,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAW7D,MAAM,MAAM,MAAM,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG;IACzC;;;;;;OAMG;IACH,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;IAE9B;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,IAAI,CAAA;KAAC,CAAC,GAAG,SAAS,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,IAAI,CAAC,CAAC,IAAI;IACpB,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC;CACxB,CAAC;AAEF;;;GAGG;AAIH,wBAAgB,MAAM,CAAC,EAAE,SAAS,SAAS,EAAE,GAAG,SAAS,SAAS,EAChE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EACpB,UAAU,GAAE,OAAO,CAAC,GAAG,CAAM,EAC7B,SAAS,GAAE,OAAO,CAAC,EAAE,CAAM,EAC3B,aAAa,GAAE,aAAkB,GAChC;IAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAA;CAAC,CAwE9C;AAED,KAAK,WAAW,CAAC,CAAC,IAAI;IACpB,MAAM,EAAE,QAAQ,CAAC;IACjB,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACtB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,wBAAgB,IAAI,CAAC,CAAC,EAAE,EAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,cAAc,EAAC,EAAE,WAAW,CAAC,CAAC,CAAC,QAsD5E;AAkBD,wBAAgB,SAAS,CAAC,CAAC,SAAS,SAAS,EAC3C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EACjB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CAEf;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EACtB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CAEf;AAmED,wBAAsB,QAAQ,CAAC,CAAC,SAAS,SAAS,EAChD,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAChB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAqCpB"}
1	+ {"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/streams.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,QAAQ,EAGR,KAAK,aAAa,EACnB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAKL,KAAK,SAAS,EACf,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAa,KAAK,SAAS,EAAC,MAAM,oCAAoC,CAAC;AAE9E,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,EAAC,YAAY,EAAE,KAAK,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAW7D,MAAM,MAAM,MAAM,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG;IACzC;;;;;;OAMG;IACH,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;IAE9B;;;;;;;;;OASG;IACH,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAEhD;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,IAAI,CAAA;KAAC,CAAC,GAAG,SAAS,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,IAAI,CAAC,CAAC,IAAI;IACpB,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC;CACxB,CAAC;AAEF;;;GAGG;AAIH,wBAAgB,MAAM,CAAC,EAAE,SAAS,SAAS,EAAE,GAAG,SAAS,SAAS,EAChE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EACpB,UAAU,GAAE,OAAO,CAAC,GAAG,CAAM,EAC7B,SAAS,GAAE,OAAO,CAAC,EAAE,CAAM,EAC3B,aAAa,GAAE,aAAkB,GAChC;IAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAA;CAAC,CAwE9C;AAED,KAAK,WAAW,CAAC,CAAC,IAAI;IACpB,MAAM,EAAE,QAAQ,CAAC;IACjB,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACtB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,wBAAgB,IAAI,CAAC,CAAC,EAAE,EAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,cAAc,EAAC,EAAE,WAAW,CAAC,CAAC,CAAC,QAsD5E;AAkBD,wBAAgB,SAAS,CAAC,CAAC,SAAS,SAAS,EAC3C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EACjB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CAEf;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EACtB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CAEf;AAmED,wBAAsB,QAAQ,CAAC,CAAC,SAAS,SAAS,EAChD,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAChB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAqCpB"}

package/out/zero-cache/src/types/streams.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"streams.js","names":["#lc","#ws","#closeStream","#messageHandler","#connected","#handleOpen","#handleClose","#handleError","#conn"],"sources":["../../../../../zero-cache/src/types/streams.ts"],"sourcesContent":["import {\n pipeline,\n Readable,\n Transform,\n Writable,\n type DuplexOptions,\n} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n createWebSocketStream,\n type CloseEvent,\n type ErrorEvent,\n type MessageEvent,\n type WebSocket,\n} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {BigIntJSON, type JSONValue} from '../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Subscription, type Options} from './subscription.ts';\nimport {\n closeWithError,\n expectPingsForLiveness,\n sendPingsForLiveness,\n} from './ws.ts';\n\n// Consistent with Postgres keepalives, and shorter than the\n// commonly used default idle timeout of 1 minute.\nconst PING_INTERVAL_MS = 30_000;\n\nexport type Source<T> = AsyncIterable<T> & {\n /*\n Immediately terminates all current iterations (i.e. {@link AsyncIterator.next next()})\n * will return `{value: undefined, done: true}`), and prevents any subsequent iterations\n * from yielding any values.\n \n @param err Terminate the iteration by throwing the `err` instead.\n /\n cancel: (err?: Error) => void;\n\n /\n The presence of a `pipeline` iterable allows the usual \"consumed-on-iterate\" semantics\n * to be overridden.\n \n This is suitable for transport layers that serialize messages across processes, such\n * as the {@link streamOut()} method; pipelining allows the transport to send messages\n * as they arrive without waiting for the previous message to be acked, streaming\n * them to the receiving process where they are presumably queued and processed without\n * a per-message ack delay. The receiving end of the transport then responds with acks\n * asynchronously as the receiving end processes the messages.\n /\n pipeline?: AsyncIterable<{value: T; consumed: () => void}> \| undefined;\n};\n\nexport type Sink<T> = {\n push(message: T): void;\n};\n\n/\n Back-pressure-aware transformation of a WebSocket into\n * upstream and downstream {@link Subscription} objects.\n /\n// TODO: Change {@link streamIn} and {@link streamOut} to use this\n// under the covers so that internal communication is also\n// responsive to backpressure.\nexport function stream<In extends JSONValue, Out extends JSONValue>(\n lc: LogContext,\n ws: WebSocket,\n inSchema: v.Type<In>,\n outOptions: Options<Out> = {},\n inOptions: Options<In> = {},\n streamOptions: DuplexOptions = {},\n): {outstream: Sink<Out>; instream: Source<In>} {\n const endpoint = ws.url ?? 'client';\n function close(err?: unknown) {\n if (ws.readyState !== ws.CLOSED && ws.readyState !== ws.CLOSING) {\n if (err) {\n closeWithError(lc, ws, err);\n } else {\n lc.info?.(`closing connection to ${endpoint}`);\n ws.close();\n }\n }\n }\n\n const instream = Subscription.create<In>({\n ...inOptions,\n cleanup: (unconsumed, err) => {\n inOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n const outstream = Subscription.create<Out>({\n ...outOptions,\n cleanup: (unconsumed, err) => {\n outOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n\n const duplex = createWebSocketStream(ws, {\n ...streamOptions,\n decodeStrings: false,\n });\n\n // Outgoing transform.\n function streamOut() {\n // Mainly used for verifying that back-pressure kicks in tests.\n duplex.on('drain', () => lc.debug?.(`drained messages to ${endpoint}`));\n\n pipeline(\n Readable.from(outstream),\n new Transform({\n objectMode: true,\n transform: (msg, _encoding, callback) =>\n callback(null, BigIntJSON.stringify(msg)),\n }),\n duplex,\n err => (err ? outstream.fail(err) : outstream.cancel()),\n );\n }\n\n if (ws.readyState === ws.CONNECTING) {\n ws.on('open', () => {\n lc.info?.(`connected to ${endpoint}`);\n streamOut();\n });\n } else {\n streamOut();\n }\n\n // Incoming transform.\n pipe({\n source: duplex,\n sink: instream,\n parse: chunk => {\n const json = BigIntJSON.parse(chunk.toString());\n return v.parse(json, inSchema, 'passthrough');\n },\n });\n\n sendPingsForLiveness(lc, ws, PING_INTERVAL_MS);\n\n return {outstream, instream};\n}\n\ntype PipeOptions<T> = {\n source: Readable;\n sink: Subscription<T>;\n parse: (buffer: Buffer) => T \| null;\n bufferMessages?: number;\n};\n\nexport function pipe<T>({source, sink, parse, bufferMessages}: PipeOptions<T>) {\n bufferMessages ??= 0;\n assert(bufferMessages >= 0, 'bufferMessages must be non-negative');\n const pending: Promise<unknown>[] = [];\n\n pipeline(\n source,\n new Writable({\n decodeStrings: false,\n write: (chunk, _encoding, callback) => {\n let msg: T \| null;\n try {\n if ((msg = parse(chunk)) === null) {\n callback();\n return;\n }\n } catch (err) {\n callback(ensureError(err));\n return;\n }\n // Inbound backpressure is exerted by unconsumed messages in the\n // subscription. A buffer can be used to allow messages to queue up in\n // in the Subscription object, which allows the consumer to \"peek\" at\n // whether there are more messages immediately available\n // (via {@link Subscription.queued}.\n const {result} = sink.push(msg);\n pending.push(result);\n void result.then(() => pending.shift());\n\n if (pending.length <= bufferMessages) {\n // immediately allow more messages\n callback();\n } else {\n // wait for the oldest result in the pending queue\n pending[0].then(\n () => callback(),\n err => callback(ensureError(err)),\n );\n }\n },\n destroy: (err, callback) => {\n if (err) {\n sink.fail(ensureError(err));\n }\n // Otherwise, final will handle the cancel.\n callback();\n },\n final: callback => {\n sink.cancel();\n callback();\n },\n }),\n err => (err ? sink.fail(err) : sink.cancel()),\n );\n}\n\nfunction ensureError(err: unknown) {\n return err instanceof Error ? err : new Error(String(err));\n}\n\nconst ackSchema = v.object({ack: v.number()});\n\ntype Ack = v.Infer<typeof ackSchema>;\n\ntype Streamed<T> = {\n /* Application-level message. /\n msg: T;\n\n /* ID used for the Ack message. /\n id: number;\n};\n\nexport function streamOut<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, BigIntJSON.stringify);\n}\n\n/\n Streams out a `Source` for which messages are already stringified JSON.\n */\nexport function streamOutStringified(\n lc: LogContext,\n source: Source<string>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, json => json);\n}\n\nasync function streamOutInternal<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n stringify: (payload: T) => string,\n): Promise<void> {\n sendPingsForLiveness(lc, sink, PING_INTERVAL_MS);\n\n const closer = WebSocketCloser.forSource(lc, sink, source);\n\n const acks = new Queue<Ack>();\n sink.addEventListener('message', ({data}) => {\n try {\n if (typeof data !== 'string') {\n throw new Error('Expected string message');\n }\n acks.enqueue(v.parse(JSON.parse(data), ackSchema));\n } catch (e) {\n lc.error?.(`error parsing ack`, e);\n closer.close(e);\n }\n });\n\n try {\n let nextID = 0;\n const {pipeline} = source;\n if (pipeline) {\n lc.debug?.(`started pipelined outbound stream`);\n for await (const {value: msg, consumed} of pipeline) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`pipelining`, data);\n sink.send(data);\n\n void (async () => {\n const {ack} = await acks.dequeue();\n // lc.debug?.(`received ack`, ack);\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n consumed();\n })();\n }\n } else {\n lc.debug?.(`started synchronous outbound stream`);\n for await (const msg of source) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`sending`, data);\n sink.send(data);\n\n const {ack} = await acks.dequeue();\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n }\n }\n closer.close();\n } catch (e) {\n closer.close(e);\n }\n}\n\nexport async function streamIn<T extends JSONValue>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n): Promise<Source<T>> {\n expectPingsForLiveness(lc, source, PING_INTERVAL_MS);\n\n const streamedSchema = v.object({\n msg: schema,\n id: v.number(),\n });\n\n const sink: Subscription<T, Streamed<T>> = new Subscription<T, Streamed<T>>(\n {\n consumed: ({id}) => source.send(JSON.stringify({ack: id} satisfies Ack)),\n cleanup: () => closer.close(),\n },\n ({msg}) => msg,\n );\n\n const closer = WebSocketCloser.forSink(lc, source, sink, handleMessage);\n\n function handleMessage(event: MessageEvent) {\n const data = event.data.toString();\n if (!sink.active) {\n lc.warn?.('dropping ws message received after close', data);\n return;\n }\n try {\n const value = BigIntJSON.parse(data);\n const msg = v.parse(value, streamedSchema, 'passthrough');\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`received`, data);\n sink.push(msg);\n } catch (e) {\n closer.close(e);\n }\n }\n\n await closer.connected;\n return sink;\n}\n\nclass WebSocketCloser {\n readonly #lc: LogContext;\n readonly #ws: WebSocket;\n readonly #closeStream: () => void;\n readonly #messageHandler: ((e: MessageEvent) => void \| undefined) \| null;\n readonly #connected = resolver();\n\n get connected(): Promise<void> {\n return this.#connected.promise;\n }\n\n static forSource<T>(lc: LogContext, ws: WebSocket, stream: Source<T>) {\n // If the websocket is closed, call cancel() to notify the Source of\n // any unconsumed messages.\n return new WebSocketCloser(lc, ws, () => stream.cancel());\n }\n\n static forSink<T>(\n lc: LogContext,\n ws: WebSocket,\n stream: Subscription<T, Streamed<T>>,\n messageHandler: (e: MessageEvent) => void \| undefined,\n ) {\n // If the websocket is closed, call end() to allow the downstream Sink\n // to process any pending messages before closing the stream.\n return new WebSocketCloser(lc, ws, () => stream.end(), messageHandler);\n }\n\n private constructor(\n lc: LogContext,\n ws: WebSocket,\n closeStream: () => void,\n messageHandler?: (e: MessageEvent) => void \| undefined,\n ) {\n this.#lc = lc;\n this.#ws = ws;\n this.#closeStream = closeStream;\n this.#messageHandler = messageHandler ?? null;\n\n ws.addEventListener('open', this.#handleOpen);\n ws.addEventListener('close', this.#handleClose);\n ws.addEventListener('error', this.#handleError);\n if (this.#messageHandler) {\n ws.addEventListener('message', this.#messageHandler);\n }\n\n switch (ws.readyState) {\n case ws.CONNECTING:\n break; // expected for new connections. resolve or reject in handlers.\n case ws.OPEN:\n this.#connected.resolve();\n break;\n default:\n this.#connected.reject(\n new Error(`websocket already in state ${ws.readyState}`),\n );\n break;\n }\n }\n\n get #conn(): string {\n return 'connection' + (this.#ws.url ? ` to ${this.#ws.url}` : '');\n }\n\n #handleOpen = () => {\n this.#lc.info?.(`${this.#conn} established`);\n this.#connected.resolve();\n };\n\n #handleClose = (e: CloseEvent) => {\n const {code, reason, wasClean} = e;\n this.#lc.info?.(`${this.#conn} closed`, {\n code,\n reason,\n wasClean,\n });\n this.close();\n this.#connected.reject(`${this.#conn} closed with code ${code}`);\n };\n\n #handleError = ({message, error}: ErrorEvent) => {\n if (this.#ws.readyState === this.#ws.OPEN) {\n this.#lc.error?.(`error in ${this.#conn}`, message, error);\n }\n this.#connected.reject(error);\n };\n\n close(err?: unknown) {\n if (err) {\n this.#lc.error?.(`closing stream with error`, err);\n }\n this.#closeStream();\n if (!this.closed()) {\n this.#ws.close();\n }\n }\n\n closed() {\n return (\n this.#ws.readyState === this.#ws.CLOSED \|\|\n this.#ws.readyState === this.#ws.CLOSING\n );\n }\n}\n"],"mappings":";;;;;;;;;;AA6BA,IAAM,mBAAmB;;;;;AAqCzB,SAAgB,OACd,IACA,IACA,UACA,aAA2B,CAAC,GAC5B,YAAyB,CAAC,GAC1B,gBAA+B,CAAC,GACc;CAC9C,MAAM,WAAW,GAAG,OAAO;CAC3B,SAAS,MAAM,KAAe;EAC5B,IAAI,GAAG,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,SACtD,IAAI,KACF,eAAe,IAAI,IAAI,GAAG;OACrB;GACL,GAAG,OAAO,yBAAyB,UAAU;GAC7C,GAAG,MAAM;EACX;CAEJ;CAEA,MAAM,WAAW,aAAa,OAAW;EACvC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,UAAU,UAAU,YAAY,GAAG;GACnC,MAAM,GAAG;EACX;CACF,CAAC;CACD,MAAM,YAAY,aAAa,OAAY;EACzC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,WAAW,UAAU,YAAY,GAAG;GACpC,MAAM,GAAG;EACX;CACF,CAAC;CAED,MAAM,SAAS,sBAAsB,IAAI;EACvC,GAAG;EACH,eAAe;CACjB,CAAC;CAGD,SAAS,YAAY;EAEnB,OAAO,GAAG,eAAe,GAAG,QAAQ,uBAAuB,UAAU,CAAC;EAEtE,SACE,SAAS,KAAK,SAAS,GACvB,IAAI,UAAU;GACZ,YAAY;GACZ,YAAY,KAAK,WAAW,aAC1B,SAAS,MAAM,WAAW,UAAU,GAAG,CAAC;EAC5C,CAAC,GACD,SACA,QAAQ,MAAM,UAAU,KAAK,GAAG,IAAI,UAAU,OAAO,CACvD;CACF;CAEA,IAAI,GAAG,eAAe,GAAG,YACvB,GAAG,GAAG,cAAc;EAClB,GAAG,OAAO,gBAAgB,UAAU;EACpC,UAAU;CACZ,CAAC;MAED,UAAU;CAIZ,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,UAAS;GAEd,OAAO,MADM,WAAW,MAAM,MAAM,SAAS,CAC9B,GAAM,UAAU,aAAa;EAC9C;CACF,CAAC;CAED,qBAAqB,IAAI,IAAI,gBAAgB;CAE7C,OAAO;EAAC;EAAW;CAAQ;AAC7B;AASA,SAAgB,KAAQ,EAAC,QAAQ,MAAM,OAAO,kBAAiC;CAC7E,mBAAmB;CACnB,OAAO,kBAAkB,GAAG,qCAAqC;CACjE,MAAM,UAA8B,CAAC;CAErC,SACE,QACA,IAAI,SAAS;EACX,eAAe;EACf,QAAQ,OAAO,WAAW,aAAa;GACrC,IAAI;GACJ,IAAI;IACF,KAAK,MAAM,MAAM,KAAK,OAAO,MAAM;KACjC,SAAS;KACT;IACF;GACF,SAAS,KAAK;IACZ,SAAS,YAAY,GAAG,CAAC;IACzB;GACF;GAMA,MAAM,EAAC,WAAU,KAAK,KAAK,GAAG;GAC9B,QAAQ,KAAK,MAAM;GACnB,OAAY,WAAW,QAAQ,MAAM,CAAC;GAEtC,IAAI,QAAQ,UAAU,gBAEpB,SAAS;QAGT,QAAQ,GAAG,WACH,SAAS,IACf,QAAO,SAAS,YAAY,GAAG,CAAC,CAClC;EAEJ;EACA,UAAU,KAAK,aAAa;GAC1B,IAAI,KACF,KAAK,KAAK,YAAY,GAAG,CAAC;GAG5B,SAAS;EACX;EACA,QAAO,aAAY;GACjB,KAAK,OAAO;GACZ,SAAS;EACX;CACF,CAAC,IACD,QAAQ,MAAM,KAAK,KAAK,GAAG,IAAI,KAAK,OAAO,CAC7C;AACF;AAEA,SAAS,YAAY,KAAc;CACjC,OAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAC3D;AAEA,IAAM,YAAY,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC;AAY5C,SAAgB,UACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,MAAM,WAAW,SAAS;AACjE;;;;AAKA,SAAgB,qBACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,OAAM,SAAQ,IAAI;AACzD;AAEA,eAAe,kBACb,IACA,QACA,MACA,WACe;CACf,qBAAqB,IAAI,MAAM,gBAAgB;CAE/C,MAAM,SAAS,gBAAgB,UAAU,IAAI,MAAM,MAAM;CAEzD,MAAM,OAAO,IAAI,MAAW;CAC5B,KAAK,iBAAiB,YAAY,EAAC,WAAU;EAC3C,IAAI;GACF,IAAI,OAAO,SAAS,UAClB,MAAM,IAAI,MAAM,yBAAyB;GAE3C,KAAK,QAAQ,MAAQ,KAAK,MAAM,IAAI,GAAG,SAAS,CAAC;EACnD,SAAS,GAAG;GACV,GAAG,QAAQ,qBAAqB,CAAC;GACjC,OAAO,MAAM,CAAC;EAChB;CACF,CAAC;CAED,IAAI;EACF,IAAI,SAAS;EACb,MAAM,EAAC,aAAY;EACnB,IAAI,UAAU;GACZ,GAAG,QAAQ,mCAAmC;GAC9C,WAAW,MAAM,EAAC,OAAO,KAAK,cAAa,UAAU;IACnD,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,CAAM,YAAY;KAChB,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;KAEjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;KAEpD,SAAS;IACX,GAAG;GACL;EACF,OAAO;GACL,GAAG,QAAQ,qCAAqC;GAChD,WAAW,MAAM,OAAO,QAAQ;IAC9B,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;IACjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;GAEtD;EACF;EACA,OAAO,MAAM;CACf,SAAS,GAAG;EACV,OAAO,MAAM,CAAC;CAChB;AACF;AAEA,eAAsB,SACpB,IACA,QACA,QACoB;CACpB,uBAAuB,IAAI,QAAQ,gBAAgB;CAEnD,MAAM,iBAAiB,eAAE,OAAO;EAC9B,KAAK;EACL,IAAI,eAAE,OAAO;CACf,CAAC;CAED,MAAM,OAAqC,IAAI,aAC7C;EACE,WAAW,EAAC,SAAQ,OAAO,KAAK,KAAK,UAAU,EAAC,KAAK,GAAE,CAAe,CAAC;EACvE,eAAe,OAAO,MAAM;CAC9B,IACC,EAAC,UAAS,GACb;CAEA,MAAM,SAAS,gBAAgB,QAAQ,IAAI,QAAQ,MAAM,aAAa;CAEtE,SAAS,cAAc,OAAqB;EAC1C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,CAAC,KAAK,QAAQ;GAChB,GAAG,OAAO,4CAA4C,IAAI;GAC1D;EACF;EACA,IAAI;GAEF,MAAM,MAAM,MADE,WAAW,MAAM,IACX,GAAO,gBAAgB,aAAa;GAGxD,KAAK,KAAK,GAAG;EACf,SAAS,GAAG;GACV,OAAO,MAAM,CAAC;EAChB;CACF;CAEA,MAAM,OAAO;CACb,OAAO;AACT;AAEA,IAAM,kBAAN,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA,aAAsB,SAAS;CAE/B,IAAI,YAA2B;EAC7B,OAAO,KAAKI,WAAW;CACzB;CAEA,OAAO,UAAa,IAAgB,IAAe,QAAmB;EAGpE,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,OAAO,CAAC;CAC1D;CAEA,OAAO,QACL,IACA,IACA,QACA,gBACA;EAGA,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,IAAI,GAAG,cAAc;CACvE;CAEA,YACE,IACA,IACA,aACA,gBACA;EACA,KAAKJ,MAAM;EACX,KAAKC,MAAM;EACX,KAAKC,eAAe;EACpB,KAAKC,kBAAkB,kBAAkB;EAEzC,GAAG,iBAAiB,QAAQ,KAAKE,WAAW;EAC5C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,IAAI,KAAKJ,iBACP,GAAG,iBAAiB,WAAW,KAAKA,eAAe;EAGrD,QAAQ,GAAG,YAAX;GACE,KAAK,GAAG,YACN;GACF,KAAK,GAAG;IACN,KAAKC,WAAW,QAAQ;IACxB;GACF;IACE,KAAKA,WAAW,uBACd,IAAI,MAAM,8BAA8B,GAAG,YAAY,CACzD;IACA;EACJ;CACF;CAEA,IAAII,QAAgB;EAClB,OAAO,gBAAgB,KAAKP,IAAI,MAAM,OAAO,KAAKA,IAAI,QAAQ;CAChE;CAEA,oBAAoB;EAClB,KAAKD,IAAI,OAAO,GAAG,KAAKQ,MAAM,aAAa;EAC3C,KAAKJ,WAAW,QAAQ;CAC1B;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,KAAKJ,IAAI,OAAO,GAAG,KAAKQ,MAAM,UAAU;GACtC;GACA;GACA;EACF,CAAC;EACD,KAAK,MAAM;EACX,KAAKJ,WAAW,OAAO,GAAG,KAAKI,MAAM,oBAAoB,MAAM;CACjE;CAEA,gBAAgB,EAAC,SAAS,YAAuB;EAC/C,IAAI,KAAKP,IAAI,eAAe,KAAKA,IAAI,MACnC,KAAKD,IAAI,QAAQ,YAAY,KAAKQ,SAAS,SAAS,KAAK;EAE3D,KAAKJ,WAAW,OAAO,KAAK;CAC9B;CAEA,MAAM,KAAe;EACnB,IAAI,KACF,KAAKJ,IAAI,QAAQ,6BAA6B,GAAG;EAEnD,KAAKE,aAAa;EAClB,IAAI,CAAC,KAAK,OAAO,GACf,KAAKD,IAAI,MAAM;CAEnB;CAEA,SAAS;EACP,OACE,KAAKA,IAAI,eAAe,KAAKA,IAAI,UACjC,KAAKA,IAAI,eAAe,KAAKA,IAAI;CAErC;AACF"}
1	+ {"version":3,"file":"streams.js","names":["#lc","#ws","#closeStream","#messageHandler","#connected","#handleOpen","#handleClose","#handleError","#conn"],"sources":["../../../../../zero-cache/src/types/streams.ts"],"sourcesContent":["import {\n pipeline,\n Readable,\n Transform,\n Writable,\n type DuplexOptions,\n} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n createWebSocketStream,\n type CloseEvent,\n type ErrorEvent,\n type MessageEvent,\n type WebSocket,\n} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {BigIntJSON, type JSONValue} from '../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Subscription, type Options} from './subscription.ts';\nimport {\n closeWithError,\n expectPingsForLiveness,\n sendPingsForLiveness,\n} from './ws.ts';\n\n// Consistent with Postgres keepalives, and shorter than the\n// commonly used default idle timeout of 1 minute.\nconst PING_INTERVAL_MS = 30_000;\n\nexport type Source<T> = AsyncIterable<T> & {\n /*\n Immediately terminates all current iterations (i.e. {@link AsyncIterator.next next()})\n * will return `{value: undefined, done: true}`), and prevents any subsequent iterations\n * from yielding any values.\n \n @param err Terminate the iteration by throwing the `err` instead.\n /\n cancel: (err?: Error) => void;\n\n /\n Resolves the result of the specified `other` Promise, or a\n * resolved/rejected Promise when the Source is terminated or\n * rejects.\n \n This is similar to using `Promise.race([other, sourceDone])`, but\n * in a memory safe way, as repeatedly calling `Promise.race([ ... ])` with\n * a long-lived Promise results in leaking memory via a growing list of\n * `then()` callbacks (https://github.com/nodejs/node/issues/17469).\n /\n doneOr<R>(other: Promise<R>): Promise<void \| R>;\n\n /\n The presence of a `pipeline` iterable allows the usual \"consumed-on-iterate\" semantics\n * to be overridden.\n \n This is suitable for transport layers that serialize messages across processes, such\n * as the {@link streamOut()} method; pipelining allows the transport to send messages\n * as they arrive without waiting for the previous message to be acked, streaming\n * them to the receiving process where they are presumably queued and processed without\n * a per-message ack delay. The receiving end of the transport then responds with acks\n * asynchronously as the receiving end processes the messages.\n /\n pipeline?: AsyncIterable<{value: T; consumed: () => void}> \| undefined;\n};\n\nexport type Sink<T> = {\n push(message: T): void;\n};\n\n/\n Back-pressure-aware transformation of a WebSocket into\n * upstream and downstream {@link Subscription} objects.\n /\n// TODO: Change {@link streamIn} and {@link streamOut} to use this\n// under the covers so that internal communication is also\n// responsive to backpressure.\nexport function stream<In extends JSONValue, Out extends JSONValue>(\n lc: LogContext,\n ws: WebSocket,\n inSchema: v.Type<In>,\n outOptions: Options<Out> = {},\n inOptions: Options<In> = {},\n streamOptions: DuplexOptions = {},\n): {outstream: Sink<Out>; instream: Source<In>} {\n const endpoint = ws.url ?? 'client';\n function close(err?: unknown) {\n if (ws.readyState !== ws.CLOSED && ws.readyState !== ws.CLOSING) {\n if (err) {\n closeWithError(lc, ws, err);\n } else {\n lc.info?.(`closing connection to ${endpoint}`);\n ws.close();\n }\n }\n }\n\n const instream = Subscription.create<In>({\n ...inOptions,\n cleanup: (unconsumed, err) => {\n inOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n const outstream = Subscription.create<Out>({\n ...outOptions,\n cleanup: (unconsumed, err) => {\n outOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n\n const duplex = createWebSocketStream(ws, {\n ...streamOptions,\n decodeStrings: false,\n });\n\n // Outgoing transform.\n function streamOut() {\n // Mainly used for verifying that back-pressure kicks in tests.\n duplex.on('drain', () => lc.debug?.(`drained messages to ${endpoint}`));\n\n pipeline(\n Readable.from(outstream),\n new Transform({\n objectMode: true,\n transform: (msg, _encoding, callback) =>\n callback(null, BigIntJSON.stringify(msg)),\n }),\n duplex,\n err => (err ? outstream.fail(err) : outstream.cancel()),\n );\n }\n\n if (ws.readyState === ws.CONNECTING) {\n ws.on('open', () => {\n lc.info?.(`connected to ${endpoint}`);\n streamOut();\n });\n } else {\n streamOut();\n }\n\n // Incoming transform.\n pipe({\n source: duplex,\n sink: instream,\n parse: chunk => {\n const json = BigIntJSON.parse(chunk.toString());\n return v.parse(json, inSchema, 'passthrough');\n },\n });\n\n sendPingsForLiveness(lc, ws, PING_INTERVAL_MS);\n\n return {outstream, instream};\n}\n\ntype PipeOptions<T> = {\n source: Readable;\n sink: Subscription<T>;\n parse: (buffer: Buffer) => T \| null;\n bufferMessages?: number;\n};\n\nexport function pipe<T>({source, sink, parse, bufferMessages}: PipeOptions<T>) {\n bufferMessages ??= 0;\n assert(bufferMessages >= 0, 'bufferMessages must be non-negative');\n const pending: Promise<unknown>[] = [];\n\n pipeline(\n source,\n new Writable({\n decodeStrings: false,\n write: (chunk, _encoding, callback) => {\n let msg: T \| null;\n try {\n if ((msg = parse(chunk)) === null) {\n callback();\n return;\n }\n } catch (err) {\n callback(ensureError(err));\n return;\n }\n // Inbound backpressure is exerted by unconsumed messages in the\n // subscription. A buffer can be used to allow messages to queue up in\n // in the Subscription object, which allows the consumer to \"peek\" at\n // whether there are more messages immediately available\n // (via {@link Subscription.queued}.\n const {result} = sink.push(msg);\n pending.push(result);\n void result.then(() => pending.shift());\n\n if (pending.length <= bufferMessages) {\n // immediately allow more messages\n callback();\n } else {\n // wait for the oldest result in the pending queue\n pending[0].then(\n () => callback(),\n err => callback(ensureError(err)),\n );\n }\n },\n destroy: (err, callback) => {\n if (err) {\n sink.fail(ensureError(err));\n }\n // Otherwise, final will handle the cancel.\n callback();\n },\n final: callback => {\n sink.cancel();\n callback();\n },\n }),\n err => (err ? sink.fail(err) : sink.cancel()),\n );\n}\n\nfunction ensureError(err: unknown) {\n return err instanceof Error ? err : new Error(String(err));\n}\n\nconst ackSchema = v.object({ack: v.number()});\n\ntype Ack = v.Infer<typeof ackSchema>;\n\ntype Streamed<T> = {\n /* Application-level message. /\n msg: T;\n\n /* ID used for the Ack message. /\n id: number;\n};\n\nexport function streamOut<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, BigIntJSON.stringify);\n}\n\n/\n Streams out a `Source` for which messages are already stringified JSON.\n */\nexport function streamOutStringified(\n lc: LogContext,\n source: Source<string>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, json => json);\n}\n\nasync function streamOutInternal<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n stringify: (payload: T) => string,\n): Promise<void> {\n sendPingsForLiveness(lc, sink, PING_INTERVAL_MS);\n\n const closer = WebSocketCloser.forSource(lc, sink, source);\n\n const acks = new Queue<Ack>();\n sink.addEventListener('message', ({data}) => {\n try {\n if (typeof data !== 'string') {\n throw new Error('Expected string message');\n }\n acks.enqueue(v.parse(JSON.parse(data), ackSchema));\n } catch (e) {\n lc.error?.(`error parsing ack`, e);\n closer.close(e);\n }\n });\n\n try {\n let nextID = 0;\n const {pipeline} = source;\n if (pipeline) {\n lc.debug?.(`started pipelined outbound stream`);\n for await (const {value: msg, consumed} of pipeline) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`pipelining`, data);\n sink.send(data);\n\n void (async () => {\n const {ack} = await acks.dequeue();\n // lc.debug?.(`received ack`, ack);\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n consumed();\n })();\n }\n } else {\n lc.debug?.(`started synchronous outbound stream`);\n for await (const msg of source) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`sending`, data);\n sink.send(data);\n\n const {ack} = await acks.dequeue();\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n }\n }\n closer.close();\n } catch (e) {\n closer.close(e);\n }\n}\n\nexport async function streamIn<T extends JSONValue>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n): Promise<Source<T>> {\n expectPingsForLiveness(lc, source, PING_INTERVAL_MS);\n\n const streamedSchema = v.object({\n msg: schema,\n id: v.number(),\n });\n\n const sink: Subscription<T, Streamed<T>> = new Subscription<T, Streamed<T>>(\n {\n consumed: ({id}) => source.send(JSON.stringify({ack: id} satisfies Ack)),\n cleanup: () => closer.close(),\n },\n ({msg}) => msg,\n );\n\n const closer = WebSocketCloser.forSink(lc, source, sink, handleMessage);\n\n function handleMessage(event: MessageEvent) {\n const data = event.data.toString();\n if (!sink.active) {\n lc.warn?.('dropping ws message received after close', data);\n return;\n }\n try {\n const value = BigIntJSON.parse(data);\n const msg = v.parse(value, streamedSchema, 'passthrough');\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`received`, data);\n sink.push(msg);\n } catch (e) {\n closer.close(e);\n }\n }\n\n await closer.connected;\n return sink;\n}\n\nclass WebSocketCloser {\n readonly #lc: LogContext;\n readonly #ws: WebSocket;\n readonly #closeStream: () => void;\n readonly #messageHandler: ((e: MessageEvent) => void \| undefined) \| null;\n readonly #connected = resolver();\n\n get connected(): Promise<void> {\n return this.#connected.promise;\n }\n\n static forSource<T>(lc: LogContext, ws: WebSocket, stream: Source<T>) {\n // If the websocket is closed, call cancel() to notify the Source of\n // any unconsumed messages.\n return new WebSocketCloser(lc, ws, () => stream.cancel());\n }\n\n static forSink<T>(\n lc: LogContext,\n ws: WebSocket,\n stream: Subscription<T, Streamed<T>>,\n messageHandler: (e: MessageEvent) => void \| undefined,\n ) {\n // If the websocket is closed, call end() to allow the downstream Sink\n // to process any pending messages before closing the stream.\n return new WebSocketCloser(lc, ws, () => stream.end(), messageHandler);\n }\n\n private constructor(\n lc: LogContext,\n ws: WebSocket,\n closeStream: () => void,\n messageHandler?: (e: MessageEvent) => void \| undefined,\n ) {\n this.#lc = lc;\n this.#ws = ws;\n this.#closeStream = closeStream;\n this.#messageHandler = messageHandler ?? null;\n\n ws.addEventListener('open', this.#handleOpen);\n ws.addEventListener('close', this.#handleClose);\n ws.addEventListener('error', this.#handleError);\n if (this.#messageHandler) {\n ws.addEventListener('message', this.#messageHandler);\n }\n\n switch (ws.readyState) {\n case ws.CONNECTING:\n break; // expected for new connections. resolve or reject in handlers.\n case ws.OPEN:\n this.#connected.resolve();\n break;\n default:\n this.#connected.reject(\n new Error(`websocket already in state ${ws.readyState}`),\n );\n break;\n }\n }\n\n get #conn(): string {\n return 'connection' + (this.#ws.url ? ` to ${this.#ws.url}` : '');\n }\n\n #handleOpen = () => {\n this.#lc.info?.(`${this.#conn} established`);\n this.#connected.resolve();\n };\n\n #handleClose = (e: CloseEvent) => {\n const {code, reason, wasClean} = e;\n this.#lc.info?.(`${this.#conn} closed`, {\n code,\n reason,\n wasClean,\n });\n this.close();\n this.#connected.reject(`${this.#conn} closed with code ${code}`);\n };\n\n #handleError = ({message, error}: ErrorEvent) => {\n if (this.#ws.readyState === this.#ws.OPEN) {\n this.#lc.error?.(`error in ${this.#conn}`, message, error);\n }\n this.#connected.reject(error);\n };\n\n close(err?: unknown) {\n if (err) {\n this.#lc.error?.(`closing stream with error`, err);\n }\n this.#closeStream();\n if (!this.closed()) {\n this.#ws.close();\n }\n }\n\n closed() {\n return (\n this.#ws.readyState === this.#ws.CLOSED \|\|\n this.#ws.readyState === this.#ws.CLOSING\n );\n }\n}\n"],"mappings":";;;;;;;;;;AA6BA,IAAM,mBAAmB;;;;;AAiDzB,SAAgB,OACd,IACA,IACA,UACA,aAA2B,CAAC,GAC5B,YAAyB,CAAC,GAC1B,gBAA+B,CAAC,GACc;CAC9C,MAAM,WAAW,GAAG,OAAO;CAC3B,SAAS,MAAM,KAAe;EAC5B,IAAI,GAAG,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,SACtD,IAAI,KACF,eAAe,IAAI,IAAI,GAAG;OACrB;GACL,GAAG,OAAO,yBAAyB,UAAU;GAC7C,GAAG,MAAM;EACX;CAEJ;CAEA,MAAM,WAAW,aAAa,OAAW;EACvC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,UAAU,UAAU,YAAY,GAAG;GACnC,MAAM,GAAG;EACX;CACF,CAAC;CACD,MAAM,YAAY,aAAa,OAAY;EACzC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,WAAW,UAAU,YAAY,GAAG;GACpC,MAAM,GAAG;EACX;CACF,CAAC;CAED,MAAM,SAAS,sBAAsB,IAAI;EACvC,GAAG;EACH,eAAe;CACjB,CAAC;CAGD,SAAS,YAAY;EAEnB,OAAO,GAAG,eAAe,GAAG,QAAQ,uBAAuB,UAAU,CAAC;EAEtE,SACE,SAAS,KAAK,SAAS,GACvB,IAAI,UAAU;GACZ,YAAY;GACZ,YAAY,KAAK,WAAW,aAC1B,SAAS,MAAM,WAAW,UAAU,GAAG,CAAC;EAC5C,CAAC,GACD,SACA,QAAQ,MAAM,UAAU,KAAK,GAAG,IAAI,UAAU,OAAO,CACvD;CACF;CAEA,IAAI,GAAG,eAAe,GAAG,YACvB,GAAG,GAAG,cAAc;EAClB,GAAG,OAAO,gBAAgB,UAAU;EACpC,UAAU;CACZ,CAAC;MAED,UAAU;CAIZ,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,UAAS;GAEd,OAAO,MADM,WAAW,MAAM,MAAM,SAAS,CAC9B,GAAM,UAAU,aAAa;EAC9C;CACF,CAAC;CAED,qBAAqB,IAAI,IAAI,gBAAgB;CAE7C,OAAO;EAAC;EAAW;CAAQ;AAC7B;AASA,SAAgB,KAAQ,EAAC,QAAQ,MAAM,OAAO,kBAAiC;CAC7E,mBAAmB;CACnB,OAAO,kBAAkB,GAAG,qCAAqC;CACjE,MAAM,UAA8B,CAAC;CAErC,SACE,QACA,IAAI,SAAS;EACX,eAAe;EACf,QAAQ,OAAO,WAAW,aAAa;GACrC,IAAI;GACJ,IAAI;IACF,KAAK,MAAM,MAAM,KAAK,OAAO,MAAM;KACjC,SAAS;KACT;IACF;GACF,SAAS,KAAK;IACZ,SAAS,YAAY,GAAG,CAAC;IACzB;GACF;GAMA,MAAM,EAAC,WAAU,KAAK,KAAK,GAAG;GAC9B,QAAQ,KAAK,MAAM;GACnB,OAAY,WAAW,QAAQ,MAAM,CAAC;GAEtC,IAAI,QAAQ,UAAU,gBAEpB,SAAS;QAGT,QAAQ,GAAG,WACH,SAAS,IACf,QAAO,SAAS,YAAY,GAAG,CAAC,CAClC;EAEJ;EACA,UAAU,KAAK,aAAa;GAC1B,IAAI,KACF,KAAK,KAAK,YAAY,GAAG,CAAC;GAG5B,SAAS;EACX;EACA,QAAO,aAAY;GACjB,KAAK,OAAO;GACZ,SAAS;EACX;CACF,CAAC,IACD,QAAQ,MAAM,KAAK,KAAK,GAAG,IAAI,KAAK,OAAO,CAC7C;AACF;AAEA,SAAS,YAAY,KAAc;CACjC,OAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAC3D;AAEA,IAAM,YAAY,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC;AAY5C,SAAgB,UACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,MAAM,WAAW,SAAS;AACjE;;;;AAKA,SAAgB,qBACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,OAAM,SAAQ,IAAI;AACzD;AAEA,eAAe,kBACb,IACA,QACA,MACA,WACe;CACf,qBAAqB,IAAI,MAAM,gBAAgB;CAE/C,MAAM,SAAS,gBAAgB,UAAU,IAAI,MAAM,MAAM;CAEzD,MAAM,OAAO,IAAI,MAAW;CAC5B,KAAK,iBAAiB,YAAY,EAAC,WAAU;EAC3C,IAAI;GACF,IAAI,OAAO,SAAS,UAClB,MAAM,IAAI,MAAM,yBAAyB;GAE3C,KAAK,QAAQ,MAAQ,KAAK,MAAM,IAAI,GAAG,SAAS,CAAC;EACnD,SAAS,GAAG;GACV,GAAG,QAAQ,qBAAqB,CAAC;GACjC,OAAO,MAAM,CAAC;EAChB;CACF,CAAC;CAED,IAAI;EACF,IAAI,SAAS;EACb,MAAM,EAAC,aAAY;EACnB,IAAI,UAAU;GACZ,GAAG,QAAQ,mCAAmC;GAC9C,WAAW,MAAM,EAAC,OAAO,KAAK,cAAa,UAAU;IACnD,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,CAAM,YAAY;KAChB,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;KAEjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;KAEpD,SAAS;IACX,GAAG;GACL;EACF,OAAO;GACL,GAAG,QAAQ,qCAAqC;GAChD,WAAW,MAAM,OAAO,QAAQ;IAC9B,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;IACjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;GAEtD;EACF;EACA,OAAO,MAAM;CACf,SAAS,GAAG;EACV,OAAO,MAAM,CAAC;CAChB;AACF;AAEA,eAAsB,SACpB,IACA,QACA,QACoB;CACpB,uBAAuB,IAAI,QAAQ,gBAAgB;CAEnD,MAAM,iBAAiB,eAAE,OAAO;EAC9B,KAAK;EACL,IAAI,eAAE,OAAO;CACf,CAAC;CAED,MAAM,OAAqC,IAAI,aAC7C;EACE,WAAW,EAAC,SAAQ,OAAO,KAAK,KAAK,UAAU,EAAC,KAAK,GAAE,CAAe,CAAC;EACvE,eAAe,OAAO,MAAM;CAC9B,IACC,EAAC,UAAS,GACb;CAEA,MAAM,SAAS,gBAAgB,QAAQ,IAAI,QAAQ,MAAM,aAAa;CAEtE,SAAS,cAAc,OAAqB;EAC1C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,CAAC,KAAK,QAAQ;GAChB,GAAG,OAAO,4CAA4C,IAAI;GAC1D;EACF;EACA,IAAI;GAEF,MAAM,MAAM,MADE,WAAW,MAAM,IACX,GAAO,gBAAgB,aAAa;GAGxD,KAAK,KAAK,GAAG;EACf,SAAS,GAAG;GACV,OAAO,MAAM,CAAC;EAChB;CACF;CAEA,MAAM,OAAO;CACb,OAAO;AACT;AAEA,IAAM,kBAAN,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA,aAAsB,SAAS;CAE/B,IAAI,YAA2B;EAC7B,OAAO,KAAKI,WAAW;CACzB;CAEA,OAAO,UAAa,IAAgB,IAAe,QAAmB;EAGpE,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,OAAO,CAAC;CAC1D;CAEA,OAAO,QACL,IACA,IACA,QACA,gBACA;EAGA,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,IAAI,GAAG,cAAc;CACvE;CAEA,YACE,IACA,IACA,aACA,gBACA;EACA,KAAKJ,MAAM;EACX,KAAKC,MAAM;EACX,KAAKC,eAAe;EACpB,KAAKC,kBAAkB,kBAAkB;EAEzC,GAAG,iBAAiB,QAAQ,KAAKE,WAAW;EAC5C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,IAAI,KAAKJ,iBACP,GAAG,iBAAiB,WAAW,KAAKA,eAAe;EAGrD,QAAQ,GAAG,YAAX;GACE,KAAK,GAAG,YACN;GACF,KAAK,GAAG;IACN,KAAKC,WAAW,QAAQ;IACxB;GACF;IACE,KAAKA,WAAW,uBACd,IAAI,MAAM,8BAA8B,GAAG,YAAY,CACzD;IACA;EACJ;CACF;CAEA,IAAII,QAAgB;EAClB,OAAO,gBAAgB,KAAKP,IAAI,MAAM,OAAO,KAAKA,IAAI,QAAQ;CAChE;CAEA,oBAAoB;EAClB,KAAKD,IAAI,OAAO,GAAG,KAAKQ,MAAM,aAAa;EAC3C,KAAKJ,WAAW,QAAQ;CAC1B;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,KAAKJ,IAAI,OAAO,GAAG,KAAKQ,MAAM,UAAU;GACtC;GACA;GACA;EACF,CAAC;EACD,KAAK,MAAM;EACX,KAAKJ,WAAW,OAAO,GAAG,KAAKI,MAAM,oBAAoB,MAAM;CACjE;CAEA,gBAAgB,EAAC,SAAS,YAAuB;EAC/C,IAAI,KAAKP,IAAI,eAAe,KAAKA,IAAI,MACnC,KAAKD,IAAI,QAAQ,YAAY,KAAKQ,SAAS,SAAS,KAAK;EAE3D,KAAKJ,WAAW,OAAO,KAAK;CAC9B;CAEA,MAAM,KAAe;EACnB,IAAI,KACF,KAAKJ,IAAI,QAAQ,6BAA6B,GAAG;EAEnD,KAAKE,aAAa;EAClB,IAAI,CAAC,KAAK,OAAO,GACf,KAAKD,IAAI,MAAM;CAEnB;CAEA,SAAS;EACP,OACE,KAAKA,IAAI,eAAe,KAAKA,IAAI,UACjC,KAAKA,IAAI,eAAe,KAAKA,IAAI;CAErC;AACF"}

package/out/zero-cache/src/types/subscription.d.ts CHANGED Viewed

@@ -115,6 +115,17 @@ export declare class Subscription<T, M = T> implements Source<T>, Sink<M> {
     cancel(err?: Error): void;
     /** Fails the subscription, cleans up, and throws from any iteration. */
     fail(err: Error): void;
+    /**
+     * Resolves the result of the specified `other` Promise, or a
+     * resolved/rejected Promise when the Subscription {@link end}s,
+     * is {@link cancel}ed, or {@link fail}s.
+     *
+     * This is similar to using `Promise.race([other, subscriptionDone])`, but
+     * in a memory safe way, as repeatedly calling `Promise.race([ ... ])` with
+     * a long-lived Promise results in leaking memory via a growing list of
+     * `then()` callbacks (https://github.com/nodejs/node/issues/17469).
+     */
+    doneOr<R>(other: Promise<R>): Promise<void | R>;
     get pipeline(): AsyncIterable<{
         value: T;
         consumed: () => void;