@rocicorp/zero 1.10.0-canary.7 → 1.10.0-canary.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/shared/src/promise-race.d.ts +14 -0
- package/out/shared/src/promise-race.d.ts.map +1 -1
- package/out/shared/src/promise-race.js +32 -1
- package/out/shared/src/promise-race.js.map +1 -1
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +5 -5
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/db/migration-lite.d.ts.map +1 -1
- package/out/zero-cache/src/db/migration-lite.js +5 -3
- package/out/zero-cache/src/db/migration-lite.js.map +1 -1
- package/out/zero-cache/src/db/transaction-pool.d.ts +0 -13
- package/out/zero-cache/src/db/transaction-pool.d.ts.map +1 -1
- package/out/zero-cache/src/db/transaction-pool.js +3 -69
- package/out/zero-cache/src/db/transaction-pool.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +8 -2
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +3 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js +11 -5
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +34 -0
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +152 -42
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/types/pg.d.ts.map +1 -1
- package/out/zero-cache/src/types/pg.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +4 -9
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +2 -11
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +4 -25
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#db","#replicaVersion","#onCommitted","#onFatal","#queue","#backPressureThresholdBytes","#statementTimeoutMs","#changeLogBatchSize","#drainTimeoutMs","#cdc","#approximateQueuedBytes","#running","#readyForMore","#resetBackpressureTimeout","#backpressureTimeout","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#flushChangeLog","#trackBackfillMetadata","#maybeReleaseBackPressure","#catchup","#cookieStmts","#upsertTableMetadataStmt","#upsertColumnBackfillStmt","#tx","#released"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {getHeapStatistics} from 'node:v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {orTimeout} from '../../types/timeout.ts';\nimport {\n backfillRequestSchema,\n isDataChange,\n isSchemaChange,\n type BackfillID,\n type BackfillRequest,\n type Change,\n type DataChange,\n type Identifier,\n type SchemaChange,\n type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {\n type ChangeStreamData,\n type Commit,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {UpstreamStatusMessage} from '../change-source/protocol/current/status.ts';\nimport {\n cookieOps,\n type BackfillCookie,\n type CookieOp,\n type CookieSet,\n type TableMetadataCookie,\n} from '../replicator/change-log-cookies.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport {\n extractChangeSubstring,\n reconstructWatermarkedChange,\n serializeChangeStreamData,\n type ChangeLogEntry,\n} from './change-log-codec.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n AutoResetSignal,\n markResetRequired,\n type BackfillingColumn,\n type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n subscriber: Subscriber;\n mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n | [\n 'change',\n watermark: string,\n json: string,\n orig: Exclude<Change, DataChange> | null, // null for DataChanges\n ]\n | ['ready', callback: () => void]\n | ['subscriber', SubscriberAndMode]\n | ['abort']\n | 'stop';\n\ntype PendingTransaction = {\n pool: TransactionPool;\n preCommitWatermark: string;\n pos: number;\n startingReplicationState: Promise<ReplicationOwner>;\n ack: boolean;\n // changeLog rows buffered for the next multi-row INSERT flush.\n batch: ChangeLogRow[];\n // The most recently issued flush (or metadata) process, awaited to bound\n // pipeline depth and to order the commit-time replicationState update.\n lastFlush: Promise<unknown> | undefined;\n};\n\ntype ReplicationOwner = {\n owner: string | null;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\nexport type TuningOptions = {\n backPressureLimitHeapProportion: number;\n statementTimeoutMs: number;\n changeLogBatchSize: number;\n drainTimeoutMs?: number | undefined;\n};\n\n/**\n * A single `changeLog` row, accumulated in {@link PendingTransaction.batch}\n * and written via `json_to_recordset()` (see `#flushChangeLog()`).\n */\ntype ChangeLogRow = {\n watermark: string;\n precommit: string | null;\n pos: number;\n change: string;\n};\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n * `replicaVersion`, which is the version at which initial sync\n * was performed, and any attempts to catchup from a different\n * point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n * a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n * subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n readonly id = 'storer';\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #taskID: string;\n readonly #discoveryAddress: string;\n readonly #discoveryProtocol: string;\n readonly #db: PostgresDB;\n readonly #replicaVersion: string;\n readonly #onCommitted: (c: Commit) => void;\n readonly #onFatal: (err: Error) => void;\n readonly #queue = new Queue<QueueEntry>();\n readonly #backPressureThresholdBytes: number;\n readonly #statementTimeoutMs: number;\n readonly #changeLogBatchSize: number;\n readonly #drainTimeoutMs: number;\n\n #approximateQueuedBytes = 0;\n #running = false;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n db: PostgresDB,\n replicaVersion: string,\n onCommitted: (c: Commit | UpstreamStatusMessage) => void,\n onFatal: (err: Error) => void,\n {\n backPressureLimitHeapProportion,\n statementTimeoutMs,\n changeLogBatchSize,\n drainTimeoutMs = 30_000,\n }: TuningOptions,\n ) {\n this.#lc = lc.withContext('component', 'change-log');\n this.#shard = shard;\n this.#taskID = taskID;\n this.#discoveryAddress = discoveryAddress;\n this.#discoveryProtocol = discoveryProtocol;\n this.#db = db;\n this.#replicaVersion = replicaVersion;\n this.#onCommitted = onCommitted;\n this.#onFatal = onFatal;\n this.#statementTimeoutMs = statementTimeoutMs;\n this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);\n this.#drainTimeoutMs = drainTimeoutMs;\n\n const heapStats = getHeapStatistics();\n this.#backPressureThresholdBytes =\n (heapStats.heap_size_limit - heapStats.used_heap_size) *\n backPressureLimitHeapProportion;\n\n this.#lc.info?.(\n `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n `to absorb upstream spikes`,\n {heapStats},\n );\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n async assumeOwnership(purgeLock?: PurgeLock | null) {\n const db = this.#db;\n const owner = this.#taskID;\n const ownerAddress = this.#discoveryAddress;\n const ownerProtocol = this.#discoveryProtocol;\n // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n const addressWithProtocol =\n ownerProtocol === 'ws'\n ? ownerAddress\n : `${ownerProtocol}://${ownerAddress}`;\n this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n const start = performance.now();\n await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n );\n\n if (purgeLock) {\n // Once ownership has been assumed, any initial purge-lock preventing the\n // purging of change-log records can be released, as a change-streamer\n // that was attempting to purge records will correspondingly abort on the\n // ownership check.\n void purgeLock.release();\n }\n }\n\n /**\n * The whole of what a stream connection starts from: where to resume, what to\n * ask the change source to backfill, and the raw cookie set those requests\n * were derived from.\n *\n * All four statements run in one REPEATABLE READ transaction, so they see one\n * snapshot. That is invariant 15 — the resume watermark and the cookies must\n * come from the same store, read at the same position — and it is why the\n * cookies are read here rather than by a second call: a cookie set paired\n * with a watermark it was not folded to loses every backfill that completed\n * in the interval, and those rows are not replayable from the slot.\n *\n * `cookies` is not `backfillRequests` in another shape. The request list is\n * driven off `backfilling`, so it drops the metadata of every table with no\n * in-flight backfill — correct for starting a stream, lossy as the cookie\n * snapshot that the SQLite change log continues folding onto.\n */\n async getStartStreamInitializationParameters(): Promise<{\n lastWatermark: string;\n backfillRequests: BackfillRequest[];\n cookies: CookieSet;\n }> {\n const [[{lastWatermark}], result, tableMetadata, backfilling] = await runTx(\n this.#db,\n sql => [\n sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n // Formats a BackfillRequest using json_object_agg() to construct the\n // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n // to make it optional and possibly `null`.\n sql`\n SELECT\n json_build_object(\n 'schema', b.\"schema\",\n 'name', b.\"table\",\n 'metadata', t.\"metadata\"\n ) as \"table\",\n json_object_agg(b.\"column\", b.\"backfill\")\n as \"columns\"\n FROM ${this.#cdc('backfilling')} as b\n LEFT JOIN ${this.#cdc('tableMetadata')} as t\n ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n `,\n\n // Ordered by primary key, so that this set and the SQLite change log's\n // can be compared row for row. `COLLATE \"C\"` because that comparison is\n // against a store whose TEXT columns sort by byte: a linguistic\n // collation orders `foo-bar` and `foobar` differently than SQLite's\n // BINARY does, which would read as a divergence rather than as the\n // identical set it is.\n sql<TableMetadataCookie[]>`\n SELECT \"schema\", \"table\", \"metadata\" FROM ${this.#cdc('tableMetadata')}\n ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\"`,\n\n sql<BackfillCookie[]>`\n SELECT \"schema\", \"table\", \"column\", \"backfill\" FROM ${this.#cdc('backfilling')}\n ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\", \"column\" COLLATE \"C\"`,\n ],\n {mode: Mode.READONLY},\n );\n\n return {\n lastWatermark,\n backfillRequests: v.parse(result, backfillRequestsSchema),\n cookies: {\n tableMetadata: [...tableMetadata],\n backfilling: [...backfilling],\n },\n };\n }\n\n async getMinWatermarkForCatchup(): Promise<string | null> {\n const [{minWatermark}] = await this.#db<{minWatermark: string | null}[]>\n /*sql*/ `\n SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n return minWatermark;\n }\n\n /**\n * Returns the retained Postgres catchup bounds.\n * `minWatermark` is the oldest transaction, or `null` for a new replica.\n * `lastWatermark` is the durable head.\n */\n async getCatchupBounds(): Promise<{\n minWatermark: string | null;\n lastWatermark: string;\n }> {\n const [bounds] = await this.#db<\n {minWatermark: string | null; lastWatermark: string}[]\n > /*sql*/ `\n SELECT\n (SELECT min(watermark) FROM ${this.#cdc('changeLog')}) as \"minWatermark\",\n (SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}) as \"lastWatermark\"`;\n return bounds;\n }\n\n /**\n * Lists at most `limit` committed transaction watermarks in\n * `(afterWatermark, throughWatermark]`, in ascending order.\n */\n async listCommitWatermarks(\n afterWatermark: string,\n throughWatermark: string,\n limit: number,\n ): Promise<string[]> {\n const rows = await this.#db<{watermark: string}[]> /*sql*/ `\n SELECT watermark FROM ${this.#cdc('changeLog')}\n WHERE precommit IS NOT NULL\n AND watermark > ${afterWatermark}\n AND watermark <= ${throughWatermark}\n ORDER BY watermark\n LIMIT ${limit}`;\n return rows.map(({watermark}) => watermark);\n }\n\n /**\n * Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.\n * This query matches the subscriber catchup query.\n *\n * Both queries fail when an escaped NUL reaches `change->'tag'`.\n * Postgres cannot convert that value to text.\n */\n readCatchupRange(\n afterWatermark: string,\n throughWatermark: string,\n batchRows = 2000,\n ): AsyncIterable<ChangeLogEntry[]> {\n assert(\n Number.isSafeInteger(batchRows) && batchRows > 0,\n 'Postgres change log batch size must be a positive safe integer',\n );\n return this.#db<ChangeLogEntry[]> /*sql*/ `\n SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n WHERE watermark > ${afterWatermark}\n AND watermark <= ${throughWatermark}\n ORDER BY watermark, pos`.cursor(batchRows) as AsyncIterable<\n ChangeLogEntry[]\n >;\n }\n\n purgeRecordsBefore(watermark: string): Promise<number> {\n return runTx(this.#db, async sql => {\n // This NOWAIT pre-check is an optimization to abort the transaction\n // (and release associated resources) early.\n await sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR UPDATE NOWAIT\n `;\n // If the row is purge-locked by an incoming replication-manager, it\n // will assume ownership of the change-log before releasing the lock.\n // This DELETE blocks until the lock is released, allowing the change\n // in ownership to be reliably detected (and the transaction aborted)\n // in the subsequent check.\n const [{deleted}] = await sql<{deleted: bigint}[]>`\n -- The backup watermark can be ahead of the durable changeLog if the\n -- storer is behind but the backup replica has consumed forwarded\n -- changes. Preserve the latest durable changeLog transaction as the\n -- catchup boundary instead of assuming the backup watermark exists.\n -- The storer inserts each changeLog transaction atomically, so any\n -- durable row for a watermark implies the full transaction is durable.\n WITH keep AS (\n SELECT max(watermark) AS watermark\n FROM ${this.#cdc('changeLog')}\n ), purged AS (\n DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n AND watermark < (SELECT watermark FROM keep)\n RETURNING watermark, pos\n ) SELECT COUNT(*) as deleted FROM purged;`;\n\n const [{owner}] = await sql<ReplicationOwner[]>`\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR SHARE`;\n if (owner !== this.#taskID) {\n throw new AbortError(\n `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n );\n }\n return Number(deleted);\n });\n }\n\n /**\n * @returns The JSON stringified stream message to be sent downstream.\n */\n store(watermark: string, data: ChangeStreamData) {\n // Eagerly stringify the JSON payload to:\n // - avoid redundant stringification when fanning out to subscribers\n // - efficiently estimate the amount of memory the payload consumes\n const json = serializeChangeStreamData(data);\n this.#approximateQueuedBytes += json.length;\n\n const change = data[1];\n this.#queue.enqueue([\n 'change',\n watermark,\n json,\n isDataChange(change) ? null : change, // drop DataChanges to save memory\n ]);\n\n return json;\n }\n\n abort() {\n this.#queue.enqueue(['abort']);\n }\n\n catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n }\n\n #readyForMore: Resolver<void> | null = null;\n #backpressureTimeout: NodeJS.Timeout | undefined;\n\n readyForMore(): Promise<void> | undefined {\n if (!this.#running) {\n return undefined;\n }\n if (\n this.#readyForMore === null &&\n this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n ) {\n this.#lc.warn?.(\n `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n `\\n` +\n `To inspect changeLog backlog in your change DB:\\n` +\n ` SELECT\\n` +\n ` (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n ` change->>'tag' AS operation,\\n` +\n ` COUNT(*) AS count\\n` +\n ` FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n ` GROUP BY 1, 2\\n` +\n ` ORDER BY 3 DESC\\n` +\n ` LIMIT 20;`,\n );\n this.#readyForMore = resolver();\n this.#resetBackpressureTimeout(true);\n }\n return this.#readyForMore?.promise;\n }\n\n #resetBackpressureTimeout(reschedule: boolean) {\n clearTimeout(this.#backpressureTimeout);\n this.#backpressureTimeout = reschedule\n ? setTimeout(() => {\n this.#readyForMore?.reject(\n new AbortError(\n `No statement processed within the last ${this.#statementTimeoutMs}ms. Considering the change-log to be wedged.`,\n ),\n );\n this.#readyForMore = null;\n }, this.#statementTimeoutMs)\n : undefined;\n }\n\n #maybeReleaseBackPressure() {\n if (this.#readyForMore !== null) {\n // Wait for at least 20% of the threshold to free up.\n if (\n this.#approximateQueuedBytes <\n this.#backPressureThresholdBytes * 0.8\n ) {\n this.#lc.info?.(\n `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n );\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n this.#resetBackpressureTimeout(false);\n } else {\n this.#resetBackpressureTimeout(true);\n }\n }\n }\n\n /**\n * Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.\n *\n * Uses `json_to_recordset()` so the batch is a single JSON parameter: the\n * statement text stays constant regardless of batch size, avoiding the\n * unbounded prepared-statement variants (and Postgres memory growth) that a\n * multi-row INSERT would produce. See rocicorp/mono#3511.\n *\n * Returns (and updates) {@link PendingTransaction.lastFlush}; a no-op when the\n * batch is empty.\n */\n #flushChangeLog(tx: PendingTransaction): Promise<unknown> | undefined {\n const {batch} = tx;\n if (batch.length === 0) {\n return tx.lastFlush;\n }\n tx.batch = [];\n tx.lastFlush = tx.pool.process(sql => [\n sql`\n INSERT INTO ${this.#cdc('changeLog')} (\"watermark\", \"pos\", \"change\", \"precommit\")\n SELECT \"watermark\", \"pos\", \"change\"::json, \"precommit\"\n FROM json_to_recordset(${batch}) AS x(\n \"watermark\" TEXT,\n \"pos\" INT8,\n \"change\" TEXT,\n \"precommit\" TEXT\n )`,\n ]);\n return tx.lastFlush;\n }\n\n #stopped = promiseVoid;\n\n /**\n * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n * Once {@link run()} completes, it can be called again.\n */\n async run() {\n assert(!this.#running, `storer is already running`);\n\n const {promise: stopped, resolve: signalStopped} = resolver();\n this.#running = true;\n this.#stopped = stopped;\n\n this.#lc.info?.('starting storer');\n let err: unknown;\n try {\n await this.#processQueue();\n } catch (e) {\n err = e; // used in finally\n throw e;\n } finally {\n // Release any pending backpressure so the upstream can proceed\n if (this.#readyForMore !== null) {\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n this.#resetBackpressureTimeout(false);\n this.#cancelQueueEntries(\n this.#queue.drain().filter(entry => entry !== undefined),\n err,\n );\n this.#running = false;\n signalStopped();\n this.#lc.info?.('storer stopped');\n }\n }\n\n #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n if (queue.length === 0) {\n return;\n }\n this.#lc.info?.(\n `canceling ${queue.length} entries from the changeLog queue`,\n );\n const err = e instanceof Error ? e : new AbortError('server shutting down');\n for (const entry of queue) {\n if (entry === 'stop') {\n continue;\n }\n const type = entry[0];\n switch (type) {\n case 'subscriber': {\n // Disconnect subscribers waiting to be caught up so that they can\n // reconnect and try again.\n const {subscriber} = entry[1];\n this.#lc.info?.(`disconnecting ${subscriber.id}`);\n subscriber.fail(err);\n break;\n }\n }\n }\n }\n\n async #processQueue() {\n let tx: PendingTransaction | null = null;\n let msg: QueueEntry | false;\n\n const catchupQueue: SubscriberAndMode[] = [];\n try {\n while ((msg = await this.#queue.dequeue()) !== 'stop') {\n const [msgType] = msg;\n switch (msgType) {\n case 'ready': {\n const signalReady = msg[1];\n signalReady();\n continue;\n }\n case 'subscriber': {\n const subscriber = msg[1];\n if (tx) {\n catchupQueue.push(subscriber); // Wait for the current tx to complete.\n } else {\n await this.#startCatchup([subscriber]); // Catch up immediately.\n }\n continue;\n }\n case 'abort': {\n if (tx) {\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n }\n continue;\n }\n }\n // msgType === 'change'\n const [_, watermark, json, change] = msg;\n const tag = change?.tag;\n this.#approximateQueuedBytes -= json.length;\n\n if (tag === 'begin') {\n assert(!tx, 'received BEGIN in the middle of a transaction');\n const {promise, resolve, reject} = resolver<ReplicationOwner>();\n void promise.catch(() => {}); // handle rejections before the await\n tx = {\n pool: new TransactionPool(\n this.#lc.withContext('watermark', watermark),\n {\n mode: Mode.READ_COMMITTED,\n statementResponseTimeout: this.#statementTimeoutMs,\n },\n ),\n preCommitWatermark: watermark,\n pos: 0,\n startingReplicationState: promise,\n ack: !change.skipAck,\n batch: [],\n lastFlush: undefined,\n };\n tx.pool.run(this.#db);\n // Acquire a lock on the replicationState row to detect and/or prevent\n // a concurrent ownership change.\n void tx.pool.process(tx => {\n tx<ReplicationOwner[]> /*sql*/ `\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n ([result]) => resolve(result),\n reject,\n );\n return [];\n });\n } else {\n assert(tx, () => `received change outside of transaction: ${json}`);\n tx.pos++;\n }\n\n const entry: ChangeLogRow = {\n watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n pos: tx.pos,\n // For backwards compatibility, only the change message is stored\n // in the cdc changeLog.\n change: extractChangeSubstring(json, tag),\n };\n\n if (change !== null && isSchemaChange(change)) {\n // Schema changes carry backfill / table-metadata statements that\n // must be applied in stream order relative to the changeLog rows.\n // Flush any buffered rows first, then write this row together with\n // its metadata statements as a single unit (preserving the previous\n // per-change ordering for schema changes).\n await this.#flushChangeLog(tx);\n tx.lastFlush = tx.pool.process(sql => [\n sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n ...this.#trackBackfillMetadata(sql, change),\n ]);\n } else {\n // Accumulate plain changeLog rows (begin, data changes, commit) and\n // write them as a single multi-row INSERT. Collapsing the per-change\n // single-row INSERTs into batches is the dominant cost reduction for\n // large transactions, where the previous one-statement-per-change\n // path dominated the upstream replication lag.\n tx.batch.push(entry);\n if (tx.batch.length >= this.#changeLogBatchSize) {\n // Bound pipeline depth (and thus memory) by awaiting the previous\n // flush before issuing the next. This is the batched analog of the\n // previous per-100-statement backpressure await, and likewise\n // guards against memory blowup on very large transactions.\n const prevFlush = tx.lastFlush;\n void this.#flushChangeLog(tx);\n await prevFlush;\n }\n }\n this.#maybeReleaseBackPressure();\n\n if (tag === 'commit') {\n // Flush any remaining buffered changeLog rows (including this commit\n // row) before updating the replication state, so the state update is\n // ordered after all changeLog inserts for this transaction.\n void this.#flushChangeLog(tx);\n\n const {owner} = await tx.startingReplicationState;\n if (owner !== this.#taskID) {\n // Ownership change reflected in the replicationState read in 'begin'.\n tx.pool.fail(\n new AbortError(\n `changeLog ownership has been assumed by ${owner}`,\n ),\n );\n } else {\n // Update the replication state.\n const lastWatermark = watermark;\n void tx.pool.process(tx => [\n tx`\n UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n ]);\n tx.pool.setDone();\n }\n\n await tx.pool.done();\n\n // ACK the LSN to the upstream Postgres.\n if (tx.ack) {\n this.#onCommitted(['commit', change, {watermark}]);\n }\n tx = null;\n\n // Before beginning the next transaction, open a READONLY snapshot to\n // concurrently catchup any queued subscribers.\n await this.#startCatchup(catchupQueue.splice(0));\n } else if (tag === 'rollback') {\n // Aborted transactions are not stored in the changeLog. Abort the current tx\n // and process catchup of subscribers that were waiting for it to end.\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n\n await this.#startCatchup(catchupQueue.splice(0));\n }\n }\n } catch (e) {\n catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n }\n\n async #startCatchup(subs: SubscriberAndMode[]) {\n if (subs.length === 0) {\n return;\n }\n\n const lc = this.#lc.withContext('pool', 'catchup');\n // TODO: Consider setting initialWorkers to accomodate parallel\n // catchups of multiple subscribers. The tricky part is\n // staying within the db's max connections.\n const reader = new TransactionPool(lc, {mode: Mode.READONLY});\n reader.run(this.#db);\n\n let lastWatermark: string | undefined;\n try {\n // Ensure that the transaction has started (and is thus holding a snapshot\n // of the database) before continuing on to commit more changes. This is\n // done by performing a single read on the db, which determines the\n // snapshot for the REPEATABLE_READ transaction.\n [{lastWatermark}] = await reader.processReadTask(\n sql => sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}\n `,\n );\n } catch (e) {\n subs.map(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n\n // Run the actual catchup queries in the background. Errors are handled in\n // #catchup() by disconnecting the associated subscriber.\n void Promise.all(\n subs.map(sub =>\n this.#catchup(\n lc.withContext('subscriber', sub.subscriber.id),\n sub,\n lastWatermark,\n reader,\n ),\n ),\n ).finally(() => reader.setDone());\n }\n\n async #catchup(\n lc: LogContext,\n {subscriber: sub, mode}: SubscriberAndMode,\n lastWatermark: string,\n reader: TransactionPool,\n ) {\n try {\n lc.info?.(`starting catchup`);\n await reader.processReadTask(async tx => {\n lc.info?.(`catching up`);\n const start = Date.now();\n\n // When starting from initial-sync, there won't be a change with a watermark\n // equal to the replica version. This is the empty changeLog scenario.\n let watermarkFound = sub.watermark === this.#replicaVersion;\n let count = 0;\n let lastBatchConsumed: Promise<unknown> | undefined;\n\n for await (const entries of tx<ChangeLogEntry[]> /*sql*/ `\n SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n WHERE watermark >= ${sub.watermark}\n AND watermark <= ${lastWatermark}\n ORDER BY watermark, pos`.cursor(2000)) {\n // Wait for the last batch of entries to be consumed by the\n // subscriber before sending down the current batch. This pipelining\n // allows one batch of changes to be received from the change-db\n // while the previous batch of changes are sent to the subscriber,\n // resulting in flow control that caps the number of changes\n // referenced in memory to 2 * batch-size.\n const start = performance.now();\n await lastBatchConsumed;\n const elapsed = performance.now() - start;\n if (lastBatchConsumed) {\n lc[elapsed > 100 ? 'info' : 'debug']?.(\n `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n );\n }\n\n for (const entry of entries) {\n if (entry.watermark === sub.watermark) {\n // This should be the first entry.\n // Catchup starts from *after* the watermark.\n watermarkFound = true;\n } else if (watermarkFound) {\n lastBatchConsumed = sub.catchup(\n reconstructWatermarkedChange(entry),\n );\n count++;\n } else if (mode === 'backup') {\n throw new AutoResetSignal(\n `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n );\n } else {\n lc.warn?.(\n `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n );\n sub.close(\n ErrorType.WatermarkTooOld,\n `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n );\n return;\n }\n }\n }\n if (watermarkFound) {\n await lastBatchConsumed;\n lc.info?.(\n `caught up ${sub.id} with ${count} changes (${\n Date.now() - start\n } ms)`,\n );\n } else {\n // The subscriber is ahead of the latest durable changeLog entry\n // (lastWatermark). This can legitimately happen: changes are\n // forwarded to subscribers (the backup replica and view-syncers)\n // concurrently with — and can outrun — the durable store, so a\n // replica may briefly lead the change DB after the storer falls\n // behind or the change-streamer restarts. No catchup is possible or\n // needed; once the change DB catches back up, forwarding resumes and\n // the subscriber dedups any watermarks it already has. Unlike the\n // AutoResetSignal / WatermarkTooOld cases above, this is not a gap in\n // replication history, so the subscriber is simply marked caught up.\n lc.warn?.(\n `subscriber ${sub.id} at watermark ${sub.watermark} is ahead of ` +\n `the latest durable watermark ${lastWatermark}; waiting for the ` +\n `change DB to catch up`,\n );\n }\n // Start draining messages buffered during catchup. The returned promise\n // is intentionally not awaited here: while the drain is in progress,\n // new sends keep appending to the subscriber backlog and inherit its\n // byte-based backpressure.\n void sub.setCaughtUp();\n });\n } catch (err) {\n lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n if (err instanceof AutoResetSignal) {\n await markResetRequired(this.#db, this.#shard);\n this.#onFatal(err);\n }\n sub.fail(err);\n }\n }\n\n /**\n * Returns the db statements necessary to track backfill and table metadata\n * presented in the `change`, if any.\n *\n * This is the Postgres interpreter of {@link cookieOps}, which is also\n * interpreted against the SQLite change log by `ChangeLogCookieWriter`. The\n * two stores have to agree on every transition forever, and the failure mode\n * if they drift is a backfill that is silently never re-requested — so the\n * decision of what a schema change *means* lives in one place and only its\n * transport lives here.\n */\n #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n return cookieOps(change).flatMap(op => this.#cookieStmts(sql, op));\n }\n\n #cookieStmts(sql: PostgresTransaction, op: CookieOp): PendingQuery<Row[]>[] {\n switch (op.op) {\n case 'upsert-metadata':\n return [this.#upsertTableMetadataStmt(sql, op.table, op.metadata)];\n\n case 'upsert-backfill':\n return [\n this.#upsertColumnBackfillStmt(sql, op.table, op.column, op.backfill),\n ];\n\n case 'rename-table': {\n const {old} = op;\n const row = {schema: op.new.schema, table: op.new.name};\n return [\n sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n ];\n }\n\n case 'drop-table': {\n const {schema, name} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('tableMetadata')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n ];\n }\n\n case 'rename-column': {\n const {schema, name: table} = op.table;\n return [\n sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${op.new}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${op.old}`,\n ];\n }\n\n case 'drop-column': {\n const {schema, name} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${op.column}`,\n ];\n }\n\n case 'complete-backfill': {\n const {schema, name: table} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql([...op.columns])}`,\n ];\n }\n\n default:\n unreachable(op);\n }\n }\n\n #upsertTableMetadataStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n metadata: TableMetadata,\n ) {\n const row: TableMetadataRow = {schema, table, metadata};\n return sql`\n INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\")\n DO UPDATE SET ${sql(row)};\n `;\n }\n\n #upsertColumnBackfillStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n column: string,\n backfill: BackfillID,\n ) {\n const row: BackfillingColumn = {schema, table, column, backfill};\n return sql`\n INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\", \"column\")\n DO UPDATE SET ${sql(row)};\n `;\n }\n\n /**\n * Waits until all currently queued entries have been processed.\n * This is only used in tests.\n */\n async allProcessed() {\n if (this.#running) {\n const {promise, resolve} = resolver();\n this.#queue.enqueue(['ready', resolve]);\n await promise;\n }\n }\n\n /**\n * Stops the storer and waits up to a drain timeout for entries to drain,\n * throwing an exception if it doesn't drain in time, in order to abort\n * the server when PG (or the connection to it) appears to be wedged.\n */\n async stop() {\n if (this.#running) {\n this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n this.abort(); // for cleanliness, abort any open transactions\n this.#queue.enqueue('stop');\n }\n if (\n (await orTimeout(this.#stopped, this.#drainTimeoutMs)) === 'timed-out'\n ) {\n throw new AbortError(\n `changeLog did not drain within ${this.#drainTimeoutMs}ms`,\n );\n }\n }\n}\n\nexport class PurgeLock {\n readonly #lc: LogContext;\n readonly #tx: TransactionPool;\n readonly replicaVersion: string;\n readonly minWatermark: string;\n\n constructor(\n lc: LogContext,\n tx: TransactionPool,\n replicaVersion: string,\n watermark: string,\n ) {\n this.#lc = lc;\n this.#tx = tx;\n this.replicaVersion = replicaVersion;\n this.minWatermark = watermark;\n }\n\n #released = false;\n\n async release() {\n if (this.#released) {\n return;\n }\n this.#released = true;\n this.#tx.setDone();\n await this.#tx\n .done()\n .catch(e => this.#lc.warn?.(`error from purge-lock release`, e));\n this.#lc.info?.(`released purge lock on ${this.minWatermark}`);\n }\n}\n\nexport class PurgeLocker {\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #db: PostgresDB;\n\n constructor(lc: LogContext, shard: ShardID, db: PostgresDB) {\n this.#lc = lc.withContext('component', 'purge-locker');\n this.#shard = shard;\n this.#db = db;\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n async acquire() {\n const tx = new TransactionPool(this.#lc, {mode: Mode.READ_COMMITTED}).run(\n this.#db,\n );\n const row = await tx.processReadTask(\n sql => sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR SHARE \n `,\n );\n if (row.length === 0) {\n this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);\n tx.setDone();\n await tx.done();\n return null;\n }\n const [{watermark}] = row;\n const [{replicaVersion}] = await tx.processReadTask(\n sql => sql<{replicaVersion: string}[]>`\n SELECT \"replicaVersion\" FROM ${this.#cdc('replicationConfig')}\n `,\n );\n this.#lc.info?.(\n `locked watermark ${watermark} from being purged from replica@${replicaVersion}`,\n );\n return new PurgeLock(this.#lc, tx, replicaVersion, watermark);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA0FA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkD5D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,MAAkB;CACxC;CACA;CACA;CACA;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,aACA,SACA,EACE,iCACA,oBACA,oBACA,iBAAiB,OAEnB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKC,SAAS;EACd,KAAKC,UAAU;EACf,KAAKC,oBAAoB;EACzB,KAAKC,qBAAqB;EAC1B,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;EACpB,KAAKC,WAAW;EAChB,KAAKG,sBAAsB;EAC3B,KAAKC,sBAAsB,KAAK,IAAI,GAAG,kBAAkB;EACzD,KAAKC,kBAAkB;EAEvB,MAAM,YAAY,kBAAkB;EACpC,KAAKH,+BACF,UAAU,kBAAkB,UAAU,kBACvC;EAEF,KAAKV,IAAI,OACP,gBAAgB,KAAKU,8BAA8B,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAE/E,EAAC,UAAS,CACZ;CACF;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKL,IAAI,GAAG,UAAU,KAAKJ,MAAM,EAAE,GAAG,OAAO;CACtD;CAEA,MAAM,gBAAgB,WAA8B;EAClD,MAAM,KAAK,KAAKI;EAChB,MAAM,QAAQ,KAAKH;EACnB,MAAM,eAAe,KAAKC;EAC1B,MAAM,gBAAgB,KAAKC;EAE3B,MAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,cAAc,KAAK;EAC5B,KAAKJ,IAAI,OAAO,yBAAyB,qBAAqB;EAC9D,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,EAAE,UAAU,KAAKc,KAAK,kBAAkB,EAAE,OAAO,GAAG;GAAC;GAAO,cAAc;EAAmB,CAAC;EACpG,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,KAAKd,IAAI,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,KAC1D;EAEA,IAAI,WAKF,UAAe,QAAQ;CAE3B;;;;;;;;;;;;;;;;;;CAmBA,MAAM,yCAIH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,QAAQ,eAAe,eAAe,MAAM,MACpE,KAAKK,MACL,QAAO;GACL,GAA8B;sCACA,KAAKS,KAAK,kBAAkB;GAK1D,GAAG;;;;;;;;;iBASM,KAAKA,KAAK,aAAa,EAAE;sBACpB,KAAKA,KAAK,eAAe,EAAE;;;;GAWzC,GAA0B;oDACkB,KAAKA,KAAK,eAAe,EAAE;;GAGvE,GAAqB;8DACiC,KAAKA,KAAK,aAAa,EAAE;;EAEjF,GACA,EAAC,MAAM,SAAa,CACtB;EAEA,OAAO;GACL;GACA,kBAAkB,MAAQ,QAAQ,sBAAsB;GACxD,SAAS;IACP,eAAe,CAAC,GAAG,aAAa;IAChC,aAAa,CAAC,GAAG,WAAW;GAC9B;EACF;CACF;CAEA,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,KAAKT,GAC5B;qDACyC,KAAKS,KAAK,WAAW;EACtE,OAAO;CACT;;;;;;CAOA,MAAM,mBAGH;EACD,MAAM,CAAC,UAAU,MAAM,KAAKT,GAElB;;sCAEwB,KAAKS,KAAK,WAAW,EAAE;uCACtB,KAAKA,KAAK,kBAAkB,EAAE;EACjE,OAAO;CACT;;;;;CAMA,MAAM,qBACJ,gBACA,kBACA,OACmB;EAQnB,QAAO,MAPY,KAAKT,GAAmC;8BACjC,KAAKS,KAAK,WAAW,EAAE;;2BAE1B,eAAe;4BACd,iBAAiB;;eAE9B,SACC,KAAK,EAAC,gBAAe,SAAS;CAC5C;;;;;;;;CASA,iBACE,gBACA,kBACA,YAAY,KACqB;EACjC,OACE,OAAO,cAAc,SAAS,KAAK,YAAY,GAC/C,gEACF;EACA,OAAO,KAAKT,GAA8B;kEACoB,KAAKS,KAAK,WAAW,EAAE;2BAC9D,eAAe;4BACd,iBAAiB;gCACb,OAAO,SAAS;CAG9C;CAEA,mBAAmB,WAAoC;EACrD,OAAO,MAAM,KAAKT,KAAK,OAAM,QAAO;GAGlC,MAAM,GAA0B;kCACJ,KAAKS,KAAK,WAAW,EAAE;;;;GASnD,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;;;;;;;;iBASvC,KAAKA,KAAK,WAAW,EAAE;;wBAEhB,KAAKA,KAAK,WAAW,EAAE,qBAAqB,UAAU;;;;GAKxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;8BACvB,KAAKA,KAAK,kBAAkB,EAAE;GACtD,IAAI,UAAU,KAAKZ,SACjB,MAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,OAClF;GAEF,OAAO,OAAO,OAAO;EACvB,CAAC;CACH;;;;CAKA,MAAM,WAAmB,MAAwB;EAI/C,MAAM,OAAO,0BAA0B,IAAI;EAC3C,KAAKa,2BAA2B,KAAK;EAErC,MAAM,SAAS,KAAK;EACpB,KAAKN,OAAO,QAAQ;GAClB;GACA;GACA;GACA,aAAa,MAAM,IAAI,OAAO;EAChC,CAAC;EAED,OAAO;CACT;CAEA,QAAQ;EACN,KAAKA,OAAO,QAAQ,CAAC,OAAO,CAAC;CAC/B;CAEA,QAAQ,YAAwB,MAAsB;EACpD,KAAKA,OAAO,QAAQ,CAAC,cAAc;GAAC;GAAY;EAAI,CAAC,CAAC;CACxD;CAEA,gBAAuC;CACvC;CAEA,eAA0C;EACxC,IAAI,CAAC,KAAKO,UACR;EAEF,IACE,KAAKC,kBAAkB,QACvB,KAAKF,0BAA0B,KAAKL,6BACpC;GACA,KAAKV,IAAI,OACP,+BAA+B,KAAKS,OAAO,KAAK,EAAE,qBAAqB,KAAKM,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,2SAW9H;GACA,KAAKE,gBAAgB,SAAS;GAC9B,KAAKC,0BAA0B,IAAI;EACrC;EACA,OAAO,KAAKD,eAAe;CAC7B;CAEA,0BAA0B,YAAqB;EAC7C,aAAa,KAAKE,oBAAoB;EACtC,KAAKA,uBAAuB,aACxB,iBAAiB;GACf,KAAKF,eAAe,OAClB,IAAI,WACF,0CAA0C,KAAKN,oBAAoB,6CACrE,CACF;GACA,KAAKM,gBAAgB;EACvB,GAAG,KAAKN,mBAAmB,IAC3B,KAAA;CACN;CAEA,4BAA4B;EAC1B,IAAI,KAAKM,kBAAkB,MAEzB,IACE,KAAKF,0BACL,KAAKL,8BAA8B,IACnC;GACA,KAAKV,IAAI,OACP,gCAAgC,KAAKS,OAAO,KAAK,EAAE,qBAAqB,KAAKM,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,KAC/H;GACA,KAAKE,cAAc,QAAQ;GAC3B,KAAKA,gBAAgB;GACrB,KAAKC,0BAA0B,KAAK;EACtC,OACE,KAAKA,0BAA0B,IAAI;CAGzC;;;;;;;;;;;;CAaA,gBAAgB,IAAsD;EACpE,MAAM,EAAC,UAAS;EAChB,IAAI,MAAM,WAAW,GACnB,OAAO,GAAG;EAEZ,GAAG,QAAQ,CAAC;EACZ,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG;sBACa,KAAKJ,KAAK,WAAW,EAAE;;mCAEV,MAAM;;;;;YAMrC,CAAC;EACD,OAAO,GAAG;CACZ;CAEA,WAAW;;;;;CAMX,MAAM,MAAM;EACV,OAAO,CAAC,KAAKE,UAAU,2BAA2B;EAElD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,SAAS;EAC5D,KAAKA,WAAW;EAChB,KAAKI,WAAW;EAEhB,KAAKpB,IAAI,OAAO,iBAAiB;EACjC,IAAI;EACJ,IAAI;GACF,MAAM,KAAKqB,cAAc;EAC3B,SAAS,GAAG;GACV,MAAM;GACN,MAAM;EACR,UAAU;GAER,IAAI,KAAKJ,kBAAkB,MAAM;IAC/B,KAAKA,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;GACA,KAAKC,0BAA0B,KAAK;GACpC,KAAKI,oBACH,KAAKb,OAAO,MAAM,EAAE,QAAO,UAAS,UAAU,KAAA,CAAS,GACvD,GACF;GACA,KAAKO,WAAW;GAChB,cAAc;GACd,KAAKhB,IAAI,OAAO,gBAAgB;EAClC;CACF;CAEA,oBAAoB,OAAqB,GAAY;EACnD,IAAI,MAAM,WAAW,GACnB;EAEF,KAAKA,IAAI,OACP,aAAa,MAAM,OAAO,kCAC5B;EACA,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,sBAAsB;EAC1E,KAAK,MAAM,SAAS,OAAO;GACzB,IAAI,UAAU,QACZ;GAGF,QADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;KAC3B,KAAKA,IAAI,OAAO,iBAAiB,WAAW,IAAI;KAChD,WAAW,KAAK,GAAG;KACnB;IACF;GACF;EACF;CACF;CAEA,MAAMqB,gBAAgB;EACpB,IAAI,KAAgC;EACpC,IAAI;EAEJ,MAAM,eAAoC,CAAC;EAC3C,IAAI;GACF,QAAQ,MAAM,MAAM,KAAKZ,OAAO,QAAQ,OAAO,QAAQ;IACrD,MAAM,CAAC,WAAW;IAClB,QAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;MACxB,YAAY;MACZ;KACF;KACA,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;MACvB,IAAI,IACF,aAAa,KAAK,UAAU;WAE5B,MAAM,KAAKc,cAAc,CAAC,UAAU,CAAC;MAEvC;KACF;KACA,KAAK;MACH,IAAI,IAAI;OACN,GAAG,KAAK,MAAM;OACd,MAAM,GAAG,KAAK,KAAK;OACnB,KAAK;MACP;MACA;IAEJ;IAEA,MAAM,CAAC,GAAG,WAAW,MAAM,UAAU;IACrC,MAAM,MAAM,QAAQ;IACpB,KAAKR,2BAA2B,KAAK;IAErC,IAAI,QAAQ,SAAS;KACnB,OAAO,CAAC,IAAI,+CAA+C;KAC3D,MAAM,EAAC,SAAS,SAAS,WAAU,SAA2B;KAC9D,QAAa,YAAY,CAAC,CAAC;KAC3B,KAAK;MACH,MAAM,IAAI,gBACR,KAAKf,IAAI,YAAY,aAAa,SAAS,GAC3C;OACE,MAAM;OACN,0BAA0B,KAAKW;MACjC,CACF;MACA,oBAAoB;MACpB,KAAK;MACL,0BAA0B;MAC1B,KAAK,CAAC,OAAO;MACb,OAAO,CAAC;MACR,WAAW,KAAA;KACb;KACA,GAAG,KAAK,IAAI,KAAKN,GAAG;KAGpB,GAAQ,KAAK,SAAQ,OAAM;MACzB,EAA+B;gCACX,KAAKS,KAAK,kBAAkB,EAAE,aAAa,MAC5D,CAAC,YAAY,QAAQ,MAAM,GAC5B,MACF;MACA,OAAO,CAAC;KACV,CAAC;IACH,OAAO;KACL,OAAO,UAAU,2CAA2C,MAAM;KAClE,GAAG;IACL;IAEA,MAAM,QAAsB;KAC1B,WAAW,QAAQ,WAAW,YAAY,GAAG;KAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;KACtD,KAAK,GAAG;KAGR,QAAQ,uBAAuB,MAAM,GAAG;IAC1C;IAEA,IAAI,WAAW,QAAQ,eAAe,MAAM,GAAG;KAM7C,MAAM,KAAKU,gBAAgB,EAAE;KAC7B,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG,eAAe,KAAKV,KAAK,WAAW,EAAE,GAAG,IAAI,KAAK,KACrD,GAAG,KAAKW,uBAAuB,KAAK,MAAM,CAC5C,CAAC;IACH,OAAO;KAML,GAAG,MAAM,KAAK,KAAK;KACnB,IAAI,GAAG,MAAM,UAAU,KAAKb,qBAAqB;MAK/C,MAAM,YAAY,GAAG;MACrB,KAAUY,gBAAgB,EAAE;MAC5B,MAAM;KACR;IACF;IACA,KAAKE,0BAA0B;IAE/B,IAAI,QAAQ,UAAU;KAIpB,KAAUF,gBAAgB,EAAE;KAE5B,MAAM,EAAC,UAAS,MAAM,GAAG;KACzB,IAAI,UAAU,KAAKtB,SAEjB,GAAG,KAAK,KACN,IAAI,WACF,2CAA2C,OAC7C,CACF;UACK;MAEL,MAAM,gBAAgB;MACtB,GAAQ,KAAK,SAAQ,OAAM,CACzB,EAAE;qBACK,KAAKY,KAAK,kBAAkB,EAAE,OAAO,GAAG,EAAC,cAAa,CAAC,GAChE,CAAC;MACD,GAAG,KAAK,QAAQ;KAClB;KAEA,MAAM,GAAG,KAAK,KAAK;KAGnB,IAAI,GAAG,KACL,KAAKP,aAAa;MAAC;MAAU;MAAQ,EAAC,UAAS;KAAC,CAAC;KAEnD,KAAK;KAIL,MAAM,KAAKgB,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,QAAQ,YAAY;KAG7B,GAAG,KAAK,MAAM;KACd,MAAM,GAAG,KAAK,KAAK;KACnB,KAAK;KAEL,MAAM,KAAKA,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD;GACF;EACF,SAAS,GAAG;GACV,aAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GACzD,MAAM;EACR;CACF;CAEA,MAAMA,cAAc,MAA2B;EAC7C,IAAI,KAAK,WAAW,GAClB;EAGF,MAAM,KAAK,KAAKvB,IAAI,YAAY,QAAQ,SAAS;EAIjD,MAAM,SAAS,IAAI,gBAAgB,IAAI,EAAC,MAAM,SAAa,CAAC;EAC5D,OAAO,IAAI,KAAKK,GAAG;EAEnB,IAAI;EACJ,IAAI;GAKF,CAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAA8B;sCACP,KAAKS,KAAK,kBAAkB,EAAE;OAE9D;EACF,SAAS,GAAG;GACV,KAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GAC7C,MAAM;EACR;EAIA,QAAa,IACX,KAAK,KAAI,QACP,KAAKa,SACH,GAAG,YAAY,cAAc,IAAI,WAAW,EAAE,GAC9C,KACA,eACA,MACF,CACF,CACF,EAAE,cAAc,OAAO,QAAQ,CAAC;CAClC;CAEA,MAAMA,SACJ,IACA,EAAC,YAAY,KAAK,QAClB,eACA,QACA;EACA,IAAI;GACF,GAAG,OAAO,kBAAkB;GAC5B,MAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,GAAG,OAAO,aAAa;IACvB,MAAM,QAAQ,KAAK,IAAI;IAIvB,IAAI,iBAAiB,IAAI,cAAc,KAAKrB;IAC5C,IAAI,QAAQ;IACZ,IAAI;IAEJ,WAAW,MAAM,WAAW,EAA6B;sEACK,KAAKQ,KAAK,WAAW,EAAE;gCAC7D,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,GAAI,GAAG;KAOxC,MAAM,QAAQ,YAAY,IAAI;KAC9B,MAAM;KACN,MAAM,UAAU,YAAY,IAAI,IAAI;KACpC,IAAI,mBACF,GAAG,UAAU,MAAM,SAAS,WAC1B,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,IAAI,GAAG,0CAChD;KAGF,KAAK,MAAM,SAAS,SAClB,IAAI,MAAM,cAAc,IAAI,WAG1B,iBAAiB;UACZ,IAAI,gBAAgB;MACzB,oBAAoB,IAAI,QACtB,6BAA6B,KAAK,CACpC;MACA;KACF,OAAO,IAAI,SAAS,UAClB,MAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,EACvF;UACK;MACL,GAAG,OACD,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,EAC7F;MACA,IAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,EACjF;MACA;KACF;IAEJ;IACA,IAAI,gBAAgB;KAClB,MAAM;KACN,GAAG,OACD,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,IAAI,IAAI,MACd,KACH;IACF,OAWE,GAAG,OACD,cAAc,IAAI,GAAG,gBAAgB,IAAI,UAAU,4CACjB,cAAc,wCAElD;IAMF,IAAS,YAAY;GACvB,CAAC;EACH,SAAS,KAAK;GACZ,GAAG,QAAQ,sCAAsC,IAAI,MAAM,GAAG;GAC9D,IAAI,eAAe,iBAAiB;IAClC,MAAM,kBAAkB,KAAKT,KAAK,KAAKJ,MAAM;IAC7C,KAAKO,SAAS,GAAG;GACnB;GACA,IAAI,KAAK,GAAG;EACd;CACF;;;;;;;;;;;;CAaA,uBAAuB,KAA0B,QAAsB;EACrE,OAAO,UAAU,MAAM,EAAE,SAAQ,OAAM,KAAKoB,aAAa,KAAK,EAAE,CAAC;CACnE;CAEA,aAAa,KAA0B,IAAqC;EAC1E,QAAQ,GAAG,IAAX;GACE,KAAK,mBACH,OAAO,CAAC,KAAKC,yBAAyB,KAAK,GAAG,OAAO,GAAG,QAAQ,CAAC;GAEnE,KAAK,mBACH,OAAO,CACL,KAAKC,0BAA0B,KAAK,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CACtE;GAEF,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,GAAG,IAAI;KAAQ,OAAO,GAAG,IAAI;IAAI;IACtD,OAAO,CACL,GAAG,UAAU,KAAKhB,KAAK,eAAe,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,MAC3D;GACF;GAEA,KAAK,cAAc;IACjB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,eAAe,EAAE;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MACnD;GACF;GAEA,KAAK,iBAAiB;IACpB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,kBAAkB,GAAG,IAAI;mCACtC,OAAO,iBAAiB,MAAM,kBAAkB,GAAG,KAC9E;GACF;GAEA,KAAK,eAAe;IAClB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,GAAG,QAC7E;GACF;GAEA,KAAK,qBAAqB;IACxB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAC/F;GACF;GAEA,SACE,YAAY,EAAE;EAClB;CACF;CAEA,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;EAAQ;EACtD,OAAO,GAAG;sBACQ,KAAKA,KAAK,eAAe,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEnC,IAAI,GAAG,EAAE;;CAEjC;CAEA,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;EAAQ;EAC/D,OAAO,GAAG;sBACQ,KAAKA,KAAK,aAAa,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEjC,IAAI,GAAG,EAAE;;CAEjC;;;;;CAMA,MAAM,eAAe;EACnB,IAAI,KAAKE,UAAU;GACjB,MAAM,EAAC,SAAS,YAAW,SAAS;GACpC,KAAKP,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;GACtC,MAAM;EACR;CACF;;;;;;CAOA,MAAM,OAAO;EACX,IAAI,KAAKO,UAAU;GACjB,KAAKhB,IAAI,OAAO,YAAY,KAAKS,OAAO,KAAK,EAAE,mBAAmB;GAClE,KAAK,MAAM;GACX,KAAKA,OAAO,QAAQ,MAAM;EAC5B;EACA,IACG,MAAM,UAAU,KAAKW,UAAU,KAAKP,eAAe,MAAO,aAE3D,MAAM,IAAI,WACR,kCAAkC,KAAKA,gBAAgB,GACzD;CAEJ;AACF;AAEA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,gBACA,WACA;EACA,KAAKb,MAAM;EACX,KAAK+B,MAAM;EACX,KAAK,iBAAiB;EACtB,KAAK,eAAe;CACtB;CAEA,YAAY;CAEZ,MAAM,UAAU;EACd,IAAI,KAAKC,WACP;EAEF,KAAKA,YAAY;EACjB,KAAKD,IAAI,QAAQ;EACjB,MAAM,KAAKA,IACR,KAAK,EACL,OAAM,MAAK,KAAK/B,IAAI,OAAO,iCAAiC,CAAC,CAAC;EACjE,KAAKA,IAAI,OAAO,0BAA0B,KAAK,cAAc;CAC/D;AACF;AAEA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB,IAAgB;EAC1D,KAAKA,MAAM,GAAG,YAAY,aAAa,cAAc;EACrD,KAAKC,SAAS;EACd,KAAKI,MAAM;CACb;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKA,IAAI,GAAG,UAAU,KAAKJ,MAAM,EAAE,GAAG,OAAO;CACtD;CAEA,MAAM,UAAU;EACd,MAAM,KAAK,IAAI,gBAAgB,KAAKD,KAAK,EAAC,MAAM,eAAmB,CAAC,EAAE,IACpE,KAAKK,GACP;EACA,MAAM,MAAM,MAAM,GAAG,iBACnB,QAAO,GAA0B;8BACT,KAAKS,KAAK,WAAW,EAAE;;;KAIjD;EACA,IAAI,IAAI,WAAW,GAAG;GACpB,KAAKd,IAAI,OAAO,4CAA4C;GAC5D,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,MAAM,CAAC,EAAC,eAAc;EACtB,MAAM,CAAC,EAAC,oBAAmB,MAAM,GAAG,iBAClC,QAAO,GAA+B;uCACL,KAAKc,KAAK,mBAAmB,EAAE;OAElE;EACA,KAAKd,IAAI,OACP,oBAAoB,UAAU,kCAAkC,gBAClE;EACA,OAAO,IAAI,UAAU,KAAKA,KAAK,IAAI,gBAAgB,SAAS;CAC9D;AACF"}
|
|
1
|
+
{"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#db","#replicaVersion","#onCommitted","#onFatal","#queue","#backPressureThresholdBytes","#statementTimeoutMs","#changeLogBatchSize","#drainTimeoutMs","#progressMonitor","#withTimeout","#cdc","#approximateQueuedBytes","#running","#readyForMore","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#flushChangeLog","#trackBackfillMetadata","#maybeReleaseBackPressure","#catchup","#cookieStmts","#upsertTableMetadataStmt","#upsertColumnBackfillStmt","#tx","#released","#failureThresholdMs","#onFailure","#taskUpdates","#timer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {getHeapStatistics} from 'node:v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {\n BigIntJSON,\n type JSONObject,\n} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {orTimeout} from '../../types/timeout.ts';\nimport {\n backfillRequestSchema,\n isDataChange,\n isSchemaChange,\n type BackfillID,\n type BackfillRequest,\n type Change,\n type DataChange,\n type Identifier,\n type SchemaChange,\n type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {\n type ChangeStreamData,\n type Commit,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {UpstreamStatusMessage} from '../change-source/protocol/current/status.ts';\nimport {\n cookieOps,\n type BackfillCookie,\n type CookieOp,\n type CookieSet,\n type TableMetadataCookie,\n} from '../replicator/change-log-cookies.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport {\n extractChangeSubstring,\n reconstructWatermarkedChange,\n serializeChangeStreamData,\n type ChangeLogEntry,\n} from './change-log-codec.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n AutoResetSignal,\n markResetRequired,\n type BackfillingColumn,\n type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n subscriber: Subscriber;\n mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n | [\n 'change',\n watermark: string,\n json: string,\n orig: Exclude<Change, DataChange> | null, // null for DataChanges\n ]\n | ['ready', callback: () => void]\n | ['subscriber', SubscriberAndMode]\n | ['abort']\n | 'stop';\n\ntype PendingTransaction = {\n pool: TransactionPool;\n preCommitWatermark: string;\n pos: number;\n startingReplicationState: Promise<ReplicationOwner>;\n ack: boolean;\n // changeLog rows buffered for the next multi-row INSERT flush.\n batch: ChangeLogRow[];\n // The most recently issued flush (or metadata) process, awaited to bound\n // pipeline depth and to order the commit-time replicationState update.\n lastFlush: Promise<unknown> | undefined;\n};\n\ntype ReplicationOwner = {\n owner: string | null;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\nexport type TuningOptions = {\n backPressureLimitHeapProportion: number;\n statementTimeoutMs: number;\n changeLogBatchSize: number;\n drainTimeoutMs?: number | undefined;\n};\n\n/**\n * A single `changeLog` row, accumulated in {@link PendingTransaction.batch}\n * and written via `json_to_recordset()` (see `#flushChangeLog()`).\n */\ntype ChangeLogRow = {\n watermark: string;\n precommit: string | null;\n pos: number;\n change: string;\n};\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n * `replicaVersion`, which is the version at which initial sync\n * was performed, and any attempts to catchup from a different\n * point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n * a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n * subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n readonly id = 'storer';\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #taskID: string;\n readonly #discoveryAddress: string;\n readonly #discoveryProtocol: string;\n readonly #db: PostgresDB;\n readonly #replicaVersion: string;\n readonly #onCommitted: (c: Commit) => void;\n readonly #onFatal: (err: Error) => void;\n readonly #queue = new Queue<QueueEntry>();\n readonly #backPressureThresholdBytes: number;\n readonly #statementTimeoutMs: number;\n readonly #changeLogBatchSize: number;\n readonly #drainTimeoutMs: number;\n readonly #progressMonitor: ProgressMonitor;\n\n #approximateQueuedBytes = 0;\n #running = false;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n db: PostgresDB,\n replicaVersion: string,\n onCommitted: (c: Commit | UpstreamStatusMessage) => void,\n onFatal: (err: Error) => void,\n {\n backPressureLimitHeapProportion,\n statementTimeoutMs,\n changeLogBatchSize,\n drainTimeoutMs = 30_000,\n }: TuningOptions,\n ) {\n this.#lc = lc.withContext('component', 'change-log');\n this.#shard = shard;\n this.#taskID = taskID;\n this.#discoveryAddress = discoveryAddress;\n this.#discoveryProtocol = discoveryProtocol;\n this.#db = db;\n this.#replicaVersion = replicaVersion;\n this.#onCommitted = onCommitted;\n this.#onFatal = onFatal;\n this.#statementTimeoutMs = statementTimeoutMs;\n this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);\n this.#drainTimeoutMs = drainTimeoutMs;\n this.#progressMonitor = new ProgressMonitor(\n lc,\n statementTimeoutMs,\n onFatal,\n );\n\n const heapStats = getHeapStatistics();\n this.#backPressureThresholdBytes =\n (heapStats.heap_size_limit - heapStats.used_heap_size) *\n backPressureLimitHeapProportion;\n\n this.#lc.info?.(\n `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n `to absorb upstream spikes`,\n {heapStats},\n );\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n /**\n * Bounds a one-off db call (i.e. not part of the main storer loop or the\n * background catchup read, which are tracked by the ProgressMonitor\n * instead) with a plain timeout. This covers calls like\n * {@link assumeOwnership} and {@link getStartStreamInitializationParameters}\n * that are made by the caller *before* {@link run()} -- and thus before the\n * ProgressMonitor's polling starts -- as well as ones made well after,\n * where a continuously-polling watchdog would be overkill for a single\n * bounded db round trip.\n *\n * Does not cancel `promise` on timeout (there is no cancellation mechanism\n * for a postgres.js query); it just stops waiting for it, and swallows a\n * later rejection so it doesn't surface as an unhandled rejection.\n */\n async #withTimeout<T>(name: string, promise: Promise<T>): Promise<T> {\n const result = await orTimeout(promise, this.#statementTimeoutMs);\n if (result === 'timed-out') {\n void promise.catch(() => {});\n throw new AbortError(\n `${name} did not complete within ${this.#statementTimeoutMs}ms`,\n );\n }\n return result;\n }\n\n async assumeOwnership(purgeLock?: PurgeLock | null) {\n const db = this.#db;\n const owner = this.#taskID;\n const ownerAddress = this.#discoveryAddress;\n const ownerProtocol = this.#discoveryProtocol;\n // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n const addressWithProtocol =\n ownerProtocol === 'ws'\n ? ownerAddress\n : `${ownerProtocol}://${ownerAddress}`;\n this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n const start = performance.now();\n await this.#withTimeout(\n 'assume-ownership',\n db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`,\n );\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n );\n\n if (purgeLock) {\n // Once ownership has been assumed, any initial purge-lock preventing the\n // purging of change-log records can be released, as a change-streamer\n // that was attempting to purge records will correspondingly abort on the\n // ownership check.\n void purgeLock.release();\n }\n }\n\n /**\n * The whole of what a stream connection starts from: where to resume, what to\n * ask the change source to backfill, and the raw cookie set those requests\n * were derived from.\n *\n * All four statements run in one REPEATABLE READ transaction, so they see one\n * snapshot. That is invariant 15 — the resume watermark and the cookies must\n * come from the same store, read at the same position — and it is why the\n * cookies are read here rather than by a second call: a cookie set paired\n * with a watermark it was not folded to loses every backfill that completed\n * in the interval, and those rows are not replayable from the slot.\n *\n * `cookies` is not `backfillRequests` in another shape. The request list is\n * driven off `backfilling`, so it drops the metadata of every table with no\n * in-flight backfill — correct for starting a stream, lossy as the cookie\n * snapshot that the SQLite change log continues folding onto.\n */\n async getStartStreamInitializationParameters(): Promise<{\n lastWatermark: string;\n backfillRequests: BackfillRequest[];\n cookies: CookieSet;\n }> {\n const [[{lastWatermark}], result, tableMetadata, backfilling] =\n await this.#withTimeout(\n 'get-stream-params',\n runTx(\n this.#db,\n sql => [\n sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n // Formats a BackfillRequest using json_object_agg() to construct the\n // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n // to make it optional and possibly `null`.\n sql`\n SELECT\n json_build_object(\n 'schema', b.\"schema\",\n 'name', b.\"table\",\n 'metadata', t.\"metadata\"\n ) as \"table\",\n json_object_agg(b.\"column\", b.\"backfill\")\n as \"columns\"\n FROM ${this.#cdc('backfilling')} as b\n LEFT JOIN ${this.#cdc('tableMetadata')} as t\n ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n `,\n\n // Ordered by primary key, so that this set and the SQLite change log's\n // can be compared row for row. `COLLATE \"C\"` because that comparison is\n // against a store whose TEXT columns sort by byte: a linguistic\n // collation orders `foo-bar` and `foobar` differently than SQLite's\n // BINARY does, which would read as a divergence rather than as the\n // identical set it is.\n sql<TableMetadataCookie[]>`\n SELECT \"schema\", \"table\", \"metadata\" FROM ${this.#cdc('tableMetadata')}\n ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\"`,\n\n sql<BackfillCookie[]>`\n SELECT \"schema\", \"table\", \"column\", \"backfill\" FROM ${this.#cdc('backfilling')}\n ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\", \"column\" COLLATE \"C\"`,\n ],\n {mode: Mode.READONLY},\n ),\n );\n\n return {\n lastWatermark,\n backfillRequests: v.parse(result, backfillRequestsSchema),\n cookies: {\n tableMetadata: [...tableMetadata],\n backfilling: [...backfilling],\n },\n };\n }\n\n async getMinWatermarkForCatchup(): Promise<string | null> {\n const [{minWatermark}] = await this.#withTimeout(\n 'get-min-watermark',\n this.#db<{minWatermark: string | null}[]> /*sql*/ `\n SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`,\n );\n return minWatermark;\n }\n\n /**\n * Returns the retained Postgres catchup bounds.\n * `minWatermark` is the oldest transaction, or `null` for a new replica.\n * `lastWatermark` is the durable head.\n */\n async getCatchupBounds(): Promise<{\n minWatermark: string | null;\n lastWatermark: string;\n }> {\n const [bounds] = await this.#withTimeout(\n 'get-catchup-bounds',\n this.#db<{minWatermark: string | null; lastWatermark: string}[]> /*sql*/ `\n SELECT\n (SELECT min(watermark) FROM ${this.#cdc('changeLog')}) as \"minWatermark\",\n (SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}) as \"lastWatermark\"`,\n );\n return bounds;\n }\n\n /**\n * Lists at most `limit` committed transaction watermarks in\n * `(afterWatermark, throughWatermark]`, in ascending order.\n */\n async listCommitWatermarks(\n afterWatermark: string,\n throughWatermark: string,\n limit: number,\n ): Promise<string[]> {\n const rows = await this.#db<{watermark: string}[]> /*sql*/ `\n SELECT watermark FROM ${this.#cdc('changeLog')}\n WHERE precommit IS NOT NULL\n AND watermark > ${afterWatermark}\n AND watermark <= ${throughWatermark}\n ORDER BY watermark\n LIMIT ${limit}`;\n return rows.map(({watermark}) => watermark);\n }\n\n /**\n * Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.\n * This query matches the subscriber catchup query.\n *\n * Both queries fail when an escaped NUL reaches `change->'tag'`.\n * Postgres cannot convert that value to text.\n */\n readCatchupRange(\n afterWatermark: string,\n throughWatermark: string,\n batchRows = 2000,\n ): AsyncIterable<ChangeLogEntry[]> {\n assert(\n Number.isSafeInteger(batchRows) && batchRows > 0,\n 'Postgres change log batch size must be a positive safe integer',\n );\n return this.#db<ChangeLogEntry[]> /*sql*/ `\n SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n WHERE watermark > ${afterWatermark}\n AND watermark <= ${throughWatermark}\n ORDER BY watermark, pos`.cursor(batchRows) as AsyncIterable<\n ChangeLogEntry[]\n >;\n }\n\n // Note: These are the only db `await`s that are not tracked by the\n // ProgressMonitor, as the delete can legitimately block and/or take\n // an arbitrary amount of time. Luckily, this is a background call and\n // thus cannot cause the storer loop to hang.\n purgeRecordsBefore(watermark: string): Promise<number> {\n return runTx(this.#db, async sql => {\n // This NOWAIT pre-check is an optimization to abort the transaction\n // (and release associated resources) early.\n await sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR UPDATE NOWAIT\n `;\n // If the row is purge-locked by an incoming replication-manager, it\n // will assume ownership of the change-log before releasing the lock.\n // This DELETE blocks until the lock is released, allowing the change\n // in ownership to be reliably detected (and the transaction aborted)\n // in the subsequent check.\n const [{deleted}] = await sql<{deleted: bigint}[]>`\n -- The backup watermark can be ahead of the durable changeLog if the\n -- storer is behind but the backup replica has consumed forwarded\n -- changes. Preserve the latest durable changeLog transaction as the\n -- catchup boundary instead of assuming the backup watermark exists.\n -- The storer inserts each changeLog transaction atomically, so any\n -- durable row for a watermark implies the full transaction is durable.\n WITH keep AS (\n SELECT max(watermark) AS watermark\n FROM ${this.#cdc('changeLog')}\n ), purged AS (\n DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n AND watermark < (SELECT watermark FROM keep)\n RETURNING watermark, pos\n ) SELECT COUNT(*) as deleted FROM purged;`;\n\n const [{owner}] = await sql<ReplicationOwner[]>`\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR SHARE`;\n if (owner !== this.#taskID) {\n throw new AbortError(\n `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n );\n }\n return Number(deleted);\n });\n }\n\n /**\n * @returns The JSON stringified stream message to be sent downstream.\n */\n store(watermark: string, data: ChangeStreamData) {\n // Eagerly stringify the JSON payload to:\n // - avoid redundant stringification when fanning out to subscribers\n // - efficiently estimate the amount of memory the payload consumes\n const json = serializeChangeStreamData(data);\n this.#approximateQueuedBytes += json.length;\n\n const change = data[1];\n this.#queue.enqueue([\n 'change',\n watermark,\n json,\n isDataChange(change) ? null : change, // drop DataChanges to save memory\n ]);\n\n return json;\n }\n\n abort() {\n this.#queue.enqueue(['abort']);\n }\n\n catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n }\n\n #readyForMore: Resolver<void> | null = null;\n\n readyForMore(): Promise<void> | undefined {\n if (!this.#running) {\n return undefined;\n }\n if (\n this.#readyForMore === null &&\n this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n ) {\n this.#lc.warn?.(\n `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n `\\n` +\n `To inspect changeLog backlog in your change DB:\\n` +\n ` SELECT\\n` +\n ` (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n ` change->>'tag' AS operation,\\n` +\n ` COUNT(*) AS count\\n` +\n ` FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n ` GROUP BY 1, 2\\n` +\n ` ORDER BY 3 DESC\\n` +\n ` LIMIT 20;`,\n );\n this.#readyForMore = resolver();\n }\n return this.#readyForMore?.promise;\n }\n\n #maybeReleaseBackPressure() {\n if (this.#readyForMore !== null) {\n // Wait for at least 20% of the threshold to free up.\n if (\n this.#approximateQueuedBytes <\n this.#backPressureThresholdBytes * 0.8\n ) {\n this.#lc.info?.(\n `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n );\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n }\n }\n\n /**\n * Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.\n *\n * Uses `json_to_recordset()` so the batch is a single JSON parameter: the\n * statement text stays constant regardless of batch size, avoiding the\n * unbounded prepared-statement variants (and Postgres memory growth) that a\n * multi-row INSERT would produce. See rocicorp/mono#3511.\n *\n * Returns (and updates) {@link PendingTransaction.lastFlush}; a no-op when the\n * batch is empty.\n */\n #flushChangeLog(tx: PendingTransaction): Promise<unknown> | undefined {\n const {batch} = tx;\n if (batch.length === 0) {\n return tx.lastFlush;\n }\n tx.batch = [];\n tx.lastFlush = tx.pool.process(sql => [\n sql`\n INSERT INTO ${this.#cdc('changeLog')} (\"watermark\", \"pos\", \"change\", \"precommit\")\n SELECT \"watermark\", \"pos\", \"change\"::json, \"precommit\"\n FROM json_to_recordset(${batch}) AS x(\n \"watermark\" TEXT,\n \"pos\" INT8,\n \"change\" TEXT,\n \"precommit\" TEXT\n )`,\n ]);\n return tx.lastFlush;\n }\n\n #stopped = promiseVoid;\n\n /**\n * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n * Once {@link run()} completes, it can be called again.\n */\n async run() {\n assert(!this.#running, `storer is already running`);\n\n const {promise: stopped, resolve: signalStopped} = resolver();\n this.#running = true;\n this.#stopped = stopped;\n\n this.#lc.info?.('starting storer');\n this.#progressMonitor.start(); // Note: This is stopped in stop()\n let err: unknown;\n try {\n await this.#processQueue();\n } catch (e) {\n err = e; // used in finally\n throw e;\n } finally {\n // Release any pending backpressure so the upstream can proceed\n if (this.#readyForMore !== null) {\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n this.#cancelQueueEntries(\n this.#queue.drain().filter(entry => entry !== undefined),\n err,\n );\n this.#running = false;\n signalStopped();\n this.#lc.info?.('storer stopped');\n }\n }\n\n #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n if (queue.length === 0) {\n return;\n }\n this.#lc.info?.(\n `canceling ${queue.length} entries from the changeLog queue`,\n );\n const err = e instanceof Error ? e : new AbortError('server shutting down');\n for (const entry of queue) {\n if (entry === 'stop') {\n continue;\n }\n const type = entry[0];\n switch (type) {\n case 'subscriber': {\n // Disconnect subscribers waiting to be caught up so that they can\n // reconnect and try again.\n const {subscriber} = entry[1];\n this.#lc.info?.(`disconnecting ${subscriber.id}`);\n subscriber.fail(err);\n break;\n }\n }\n }\n }\n\n async #processQueue() {\n let tx: PendingTransaction | null = null;\n let msg: QueueEntry | false;\n\n // Track the progress of each (previous and next) queue entry before it is\n // processed in the loop.\n let lastTaskDone: TaskDoneFn | undefined;\n const nextMessage = async () => {\n lastTaskDone?.();\n lastTaskDone = undefined;\n const entry = await this.#queue.dequeue();\n if (entry !== 'stop') {\n lastTaskDone = this.#progressMonitor.trackTask({\n task: 'queue-entry',\n // Note: TaskKeys are logged, so only include entry[0] (the type) to\n // avoid logging application data.\n type: entry[0],\n });\n }\n return entry;\n };\n\n const catchupQueue: SubscriberAndMode[] = [];\n try {\n while ((msg = await nextMessage()) !== 'stop') {\n const [msgType] = msg;\n switch (msgType) {\n case 'ready': {\n const signalReady = msg[1];\n signalReady();\n continue;\n }\n case 'subscriber': {\n const subscriber = msg[1];\n if (tx) {\n catchupQueue.push(subscriber); // Wait for the current tx to complete.\n } else {\n await this.#startCatchup([subscriber]); // Catch up immediately.\n }\n continue;\n }\n case 'abort': {\n if (tx) {\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n }\n continue;\n }\n }\n // msgType === 'change'\n const [_, watermark, json, change] = msg;\n const tag = change?.tag;\n this.#approximateQueuedBytes -= json.length;\n\n if (tag === 'begin') {\n assert(!tx, 'received BEGIN in the middle of a transaction');\n const {promise, resolve, reject} = resolver<ReplicationOwner>();\n void promise.catch(() => {}); // handle rejections before the await\n tx = {\n pool: new TransactionPool(\n this.#lc.withContext('watermark', watermark),\n {mode: Mode.READ_COMMITTED},\n ),\n preCommitWatermark: watermark,\n pos: 0,\n startingReplicationState: promise,\n ack: !change.skipAck,\n batch: [],\n lastFlush: undefined,\n };\n tx.pool.run(this.#db);\n // Acquire a lock on the replicationState row to detect and/or prevent\n // a concurrent ownership change.\n void tx.pool.process(tx => {\n tx<ReplicationOwner[]> /*sql*/ `\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n ([result]) => resolve(result),\n reject,\n );\n return [];\n });\n } else {\n assert(tx, () => `received change outside of transaction: ${json}`);\n tx.pos++;\n }\n\n const entry: ChangeLogRow = {\n watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n pos: tx.pos,\n // For backwards compatibility, only the change message is stored\n // in the cdc changeLog.\n change: extractChangeSubstring(json, tag),\n };\n\n if (change !== null && isSchemaChange(change)) {\n // Schema changes carry backfill / table-metadata statements that\n // must be applied in stream order relative to the changeLog rows.\n // Flush any buffered rows first, then write this row together with\n // its metadata statements as a single unit (preserving the previous\n // per-change ordering for schema changes).\n await this.#flushChangeLog(tx);\n tx.lastFlush = tx.pool.process(sql => [\n sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n ...this.#trackBackfillMetadata(sql, change),\n ]);\n } else {\n // Accumulate plain changeLog rows (begin, data changes, commit) and\n // write them as a single multi-row INSERT. Collapsing the per-change\n // single-row INSERTs into batches is the dominant cost reduction for\n // large transactions, where the previous one-statement-per-change\n // path dominated the upstream replication lag.\n tx.batch.push(entry);\n if (tx.batch.length >= this.#changeLogBatchSize) {\n // Bound pipeline depth (and thus memory) by awaiting the previous\n // flush before issuing the next. This is the batched analog of the\n // previous per-100-statement backpressure await, and likewise\n // guards against memory blowup on very large transactions.\n const prevFlush = tx.lastFlush;\n void this.#flushChangeLog(tx);\n await prevFlush;\n }\n }\n this.#maybeReleaseBackPressure();\n\n if (tag === 'commit') {\n // Flush any remaining buffered changeLog rows (including this commit\n // row) before updating the replication state, so the state update is\n // ordered after all changeLog inserts for this transaction.\n void this.#flushChangeLog(tx);\n\n const {owner} = await tx.startingReplicationState;\n if (owner !== this.#taskID) {\n // Ownership change reflected in the replicationState read in 'begin'.\n tx.pool.fail(\n new AbortError(\n `changeLog ownership has been assumed by ${owner}`,\n ),\n );\n } else {\n // Update the replication state.\n const lastWatermark = watermark;\n void tx.pool.process(tx => [\n tx`\n UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n ]);\n tx.pool.setDone();\n }\n\n await tx.pool.done();\n\n // ACK the LSN to the upstream Postgres.\n if (tx.ack) {\n this.#onCommitted(['commit', change, {watermark}]);\n }\n tx = null;\n\n // Before beginning the next transaction, open a READONLY snapshot to\n // concurrently catchup any queued subscribers.\n await this.#startCatchup(catchupQueue.splice(0));\n } else if (tag === 'rollback') {\n // Aborted transactions are not stored in the changeLog. Abort the current tx\n // and process catchup of subscribers that were waiting for it to end.\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n\n await this.#startCatchup(catchupQueue.splice(0));\n }\n }\n } catch (e) {\n catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n }\n\n async #startCatchup(subs: SubscriberAndMode[]) {\n if (subs.length === 0) {\n return;\n }\n\n const lc = this.#lc.withContext('pool', 'catchup');\n // TODO: Consider setting initialWorkers to accomodate parallel\n // catchups of multiple subscribers. The tricky part is\n // staying within the db's max connections.\n const reader = new TransactionPool(lc, {mode: Mode.READONLY});\n reader.run(this.#db);\n\n let lastWatermark: string | undefined;\n const catchupSnapshotted = this.#progressMonitor.trackTask({\n task: 'capture-catchup-snapshot',\n subscribers: subs.map(({subscriber: s}) => s.id),\n });\n try {\n // Ensure that the transaction has started (and is thus holding a snapshot\n // of the database) before continuing on to commit more changes. This is\n // done by performing a single read on the db, which determines the\n // snapshot for the REPEATABLE_READ transaction.\n [{lastWatermark}] = await reader.processReadTask(\n sql => sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}\n `,\n );\n } catch (e) {\n subs.map(({subscriber}) => subscriber.fail(e));\n throw e;\n } finally {\n catchupSnapshotted();\n }\n\n // Run the actual catchup queries in the background. Errors are handled in\n // #catchup() by disconnecting the associated subscriber.\n void Promise.all(\n subs.map(sub =>\n this.#catchup(\n lc.withContext('subscriber', sub.subscriber.id),\n sub,\n lastWatermark,\n reader,\n ),\n ),\n ).finally(() => reader.setDone());\n }\n\n async #catchup(\n lc: LogContext,\n {subscriber: sub, mode}: SubscriberAndMode,\n lastWatermark: string,\n reader: TransactionPool,\n ) {\n let entriesReceived = 0;\n let catchupProgressed = this.#progressMonitor.trackTask({\n task: 'catchup',\n subscriber: sub.id,\n entriesReceived,\n });\n try {\n lc.info?.(`starting catchup`);\n await reader.processReadTask(async tx => {\n lc.info?.(`catching up`);\n const start = Date.now();\n\n // When starting from initial-sync, there won't be a change with a watermark\n // equal to the replica version. This is the empty changeLog scenario.\n let watermarkFound = sub.watermark === this.#replicaVersion;\n let count = 0;\n let lastBatchConsumed: Promise<unknown> | undefined;\n\n for await (const entries of tx<ChangeLogEntry[]> /*sql*/ `\n SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n WHERE watermark >= ${sub.watermark}\n AND watermark <= ${lastWatermark}\n ORDER BY watermark, pos`.cursor(2000)) {\n catchupProgressed();\n entriesReceived += entries.length;\n\n try {\n // Wait for the last batch of entries to be consumed by the\n // subscriber before sending down the current batch. This pipelining\n // allows one batch of changes to be received from the change-db\n // while the previous batch of changes are sent to the subscriber,\n // resulting in flow control that caps the number of changes\n // referenced in memory to 2 * batch-size.\n const start = performance.now();\n await lastBatchConsumed;\n const elapsed = performance.now() - start;\n if (lastBatchConsumed) {\n lc[elapsed > 100 ? 'info' : 'debug']?.(\n `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n );\n }\n\n for (const entry of entries) {\n if (entry.watermark === sub.watermark) {\n // This should be the first entry.\n // Catchup starts from *after* the watermark.\n watermarkFound = true;\n } else if (watermarkFound) {\n lastBatchConsumed = sub.catchup(\n reconstructWatermarkedChange(entry),\n );\n count++;\n } else if (mode === 'backup') {\n throw new AutoResetSignal(\n `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n );\n } else {\n lc.warn?.(\n `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n );\n sub.close(\n ErrorType.WatermarkTooOld,\n `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n );\n return;\n }\n }\n } finally {\n // Track the db's read of the next chunk of changes.\n catchupProgressed = this.#progressMonitor.trackTask({\n task: 'catchup',\n subscriber: sub.id,\n entriesReceived,\n });\n }\n }\n catchupProgressed();\n\n if (watermarkFound) {\n await lastBatchConsumed;\n lc.info?.(\n `caught up ${sub.id} with ${count} changes (${\n Date.now() - start\n } ms)`,\n );\n } else {\n // The subscriber is ahead of the latest durable changeLog entry\n // (lastWatermark). This can legitimately happen: changes are\n // forwarded to subscribers (the backup replica and view-syncers)\n // concurrently with — and can outrun — the durable store, so a\n // replica may briefly lead the change DB after the storer falls\n // behind or the change-streamer restarts. No catchup is possible or\n // needed; once the change DB catches back up, forwarding resumes and\n // the subscriber dedups any watermarks it already has. Unlike the\n // AutoResetSignal / WatermarkTooOld cases above, this is not a gap in\n // replication history, so the subscriber is simply marked caught up.\n lc.warn?.(\n `subscriber ${sub.id} at watermark ${sub.watermark} is ahead of ` +\n `the latest durable watermark ${lastWatermark}; waiting for the ` +\n `change DB to catch up`,\n );\n }\n // Start draining messages buffered during catchup. The returned promise\n // is intentionally not awaited here: while the drain is in progress,\n // new sends keep appending to the subscriber backlog and inherit its\n // byte-based backpressure.\n void sub.setCaughtUp();\n });\n } catch (err) {\n lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n if (err instanceof AutoResetSignal) {\n await markResetRequired(this.#db, this.#shard);\n this.#onFatal(err);\n }\n sub.fail(err);\n } finally {\n catchupProgressed();\n }\n }\n\n /**\n * Returns the db statements necessary to track backfill and table metadata\n * presented in the `change`, if any.\n *\n * This is the Postgres interpreter of {@link cookieOps}, which is also\n * interpreted against the SQLite change log by `ChangeLogCookieWriter`. The\n * two stores have to agree on every transition forever, and the failure mode\n * if they drift is a backfill that is silently never re-requested — so the\n * decision of what a schema change *means* lives in one place and only its\n * transport lives here.\n */\n #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n return cookieOps(change).flatMap(op => this.#cookieStmts(sql, op));\n }\n\n #cookieStmts(sql: PostgresTransaction, op: CookieOp): PendingQuery<Row[]>[] {\n switch (op.op) {\n case 'upsert-metadata':\n return [this.#upsertTableMetadataStmt(sql, op.table, op.metadata)];\n\n case 'upsert-backfill':\n return [\n this.#upsertColumnBackfillStmt(sql, op.table, op.column, op.backfill),\n ];\n\n case 'rename-table': {\n const {old} = op;\n const row = {schema: op.new.schema, table: op.new.name};\n return [\n sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n ];\n }\n\n case 'drop-table': {\n const {schema, name} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('tableMetadata')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n ];\n }\n\n case 'rename-column': {\n const {schema, name: table} = op.table;\n return [\n sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${op.new}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${op.old}`,\n ];\n }\n\n case 'drop-column': {\n const {schema, name} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${op.column}`,\n ];\n }\n\n case 'complete-backfill': {\n const {schema, name: table} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql([...op.columns])}`,\n ];\n }\n\n default:\n unreachable(op);\n }\n }\n\n #upsertTableMetadataStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n metadata: TableMetadata,\n ) {\n const row: TableMetadataRow = {schema, table, metadata};\n return sql`\n INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\")\n DO UPDATE SET ${sql(row)};\n `;\n }\n\n #upsertColumnBackfillStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n column: string,\n backfill: BackfillID,\n ) {\n const row: BackfillingColumn = {schema, table, column, backfill};\n return sql`\n INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\", \"column\")\n DO UPDATE SET ${sql(row)};\n `;\n }\n\n /**\n * Waits until all currently queued entries have been processed.\n * This is only used in tests.\n */\n async allProcessed() {\n if (this.#running) {\n const {promise, resolve} = resolver();\n this.#queue.enqueue(['ready', resolve]);\n await promise;\n }\n }\n\n /**\n * Stops the storer and waits up to a drain timeout for entries to drain,\n * throwing an exception if it doesn't drain in time, in order to abort\n * the server when PG (or the connection to it) appears to be wedged.\n */\n async stop() {\n if (this.#running) {\n this.#progressMonitor.stop();\n this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n this.abort(); // for cleanliness, abort any open transactions\n this.#queue.enqueue('stop');\n }\n if (\n (await orTimeout(this.#stopped, this.#drainTimeoutMs)) === 'timed-out'\n ) {\n throw new AbortError(\n `changeLog did not drain within ${this.#drainTimeoutMs}ms`,\n );\n }\n }\n}\n\nexport class PurgeLock {\n readonly #lc: LogContext;\n readonly #tx: TransactionPool;\n readonly replicaVersion: string;\n readonly minWatermark: string;\n\n constructor(\n lc: LogContext,\n tx: TransactionPool,\n replicaVersion: string,\n watermark: string,\n ) {\n this.#lc = lc;\n this.#tx = tx;\n this.replicaVersion = replicaVersion;\n this.minWatermark = watermark;\n }\n\n #released = false;\n\n async release() {\n if (this.#released) {\n return;\n }\n this.#released = true;\n this.#tx.setDone();\n await this.#tx\n .done()\n .catch(e => this.#lc.warn?.(`error from purge-lock release`, e));\n this.#lc.info?.(`released purge lock on ${this.minWatermark}`);\n }\n}\n\nexport class PurgeLocker {\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #db: PostgresDB;\n\n constructor(lc: LogContext, shard: ShardID, db: PostgresDB) {\n this.#lc = lc.withContext('component', 'purge-locker');\n this.#shard = shard;\n this.#db = db;\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n async acquire() {\n const tx = new TransactionPool(this.#lc, {mode: Mode.READ_COMMITTED}).run(\n this.#db,\n );\n const row = await tx.processReadTask(\n sql => sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR SHARE \n `,\n );\n if (row.length === 0) {\n this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);\n tx.setDone();\n await tx.done();\n return null;\n }\n const [{watermark}] = row;\n const [{replicaVersion}] = await tx.processReadTask(\n sql => sql<{replicaVersion: string}[]>`\n SELECT \"replicaVersion\" FROM ${this.#cdc('replicationConfig')}\n `,\n );\n this.#lc.info?.(\n `locked watermark ${watermark} from being purged from replica@${replicaVersion}`,\n );\n return new PurgeLock(this.#lc, tx, replicaVersion, watermark);\n }\n}\n\n// A TaskKey, keyed by identity, representing an db task monitored by the\n// ProgressMonitor. The contents are stringified in error messages if the\n// task fails to make progress for more than a failure threshold interval.\ntype TaskKey = JSONObject;\n\ntype TaskDoneFn = () => void;\n\n/**\n * Periodic monitor for detecting the pathological condition in which a db\n * query hangs forever because of a connection problem, such as a half-closed\n * socket.\n *\n * Use this for db calls that recur or accumulate progress within a single\n * logical operation on the main storer loop (e.g. the queue-processing loop,\n * or a multi-batch catchup read), where \"no progress for N ms\" is the\n * meaningful signal. For a one-off db call -- including ones made before\n * {@link Storer.run} starts this monitor's polling -- use a plain timeout\n * (see `Storer.#withTimeout`) instead.\n *\n * The failure handler should stop the server. The ProgressMonitor will\n * assume this and stop itself after calling it.\n *\n * Internal to the Storer, but exported for testing.\n */\nexport class ProgressMonitor {\n readonly #lc: LogContext;\n readonly #failureThresholdMs;\n readonly #onFailure: (err: Error) => void;\n readonly #taskUpdates = new Map<TaskKey, Date>();\n\n #timer: NodeJS.Timeout | undefined;\n\n constructor(\n lc: LogContext,\n failureThresholdMs: number,\n onFailure: (err: Error) => void,\n ) {\n this.#lc = lc.withContext('component', 'storer-progress-monitor');\n this.#failureThresholdMs = failureThresholdMs;\n this.#onFailure = onFailure;\n }\n\n start() {\n clearInterval(this.#timer);\n this.#timer = setInterval(\n () => this.checkTaskProgress(new Date()),\n this.#failureThresholdMs,\n ).unref();\n }\n\n // Called by an internal periodic timer, but exported for testing.\n checkTaskProgress(now: Date) {\n for (const [task, lastUpdate] of this.#taskUpdates.entries()) {\n if (now.getTime() - lastUpdate.getTime() >= this.#failureThresholdMs) {\n this.#lc.error?.(\n `Last task update was more than ${this.#failureThresholdMs}ms ago`,\n {task, lastUpdate},\n );\n this.#onFailure(\n new Error(\n `Task failed to progress for over ${this.#failureThresholdMs}ms: ${BigIntJSON.stringify(task)}`,\n ),\n );\n this.stop();\n return;\n }\n }\n }\n\n /**\n * Starts a task to be tracked.\n *\n * @return a callback to invoke when the task is done.\n */\n trackTask(task: TaskKey, now = new Date()): TaskDoneFn {\n this.#taskUpdates.set(task, now);\n return () => {\n this.#taskUpdates.delete(task);\n };\n }\n\n stop() {\n clearInterval(this.#timer);\n this.#taskUpdates.clear();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AA8FA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkD5D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,MAAkB;CACxC;CACA;CACA;CACA;CACA;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,aACA,SACA,EACE,iCACA,oBACA,oBACA,iBAAiB,OAEnB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKC,SAAS;EACd,KAAKC,UAAU;EACf,KAAKC,oBAAoB;EACzB,KAAKC,qBAAqB;EAC1B,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;EACpB,KAAKC,WAAW;EAChB,KAAKG,sBAAsB;EAC3B,KAAKC,sBAAsB,KAAK,IAAI,GAAG,kBAAkB;EACzD,KAAKC,kBAAkB;EACvB,KAAKC,mBAAmB,IAAI,gBAC1B,IACA,oBACA,OACF;EAEA,MAAM,YAAY,kBAAkB;EACpC,KAAKJ,+BACF,UAAU,kBAAkB,UAAU,kBACvC;EAEF,KAAKV,IAAI,OACP,gBAAgB,KAAKU,8BAA8B,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAE/E,EAAC,UAAS,CACZ;CACF;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKL,IAAI,GAAG,UAAU,KAAKJ,MAAM,EAAE,GAAG,OAAO;CACtD;;;;;;;;;;;;;;;CAgBA,MAAMc,aAAgB,MAAc,SAAiC;EACnE,MAAM,SAAS,MAAM,UAAU,SAAS,KAAKJ,mBAAmB;EAChE,IAAI,WAAW,aAAa;GAC1B,QAAa,YAAY,CAAC,CAAC;GAC3B,MAAM,IAAI,WACR,GAAG,KAAK,2BAA2B,KAAKA,oBAAoB,GAC9D;EACF;EACA,OAAO;CACT;CAEA,MAAM,gBAAgB,WAA8B;EAClD,MAAM,KAAK,KAAKN;EAChB,MAAM,QAAQ,KAAKH;EACnB,MAAM,eAAe,KAAKC;EAC1B,MAAM,gBAAgB,KAAKC;EAE3B,MAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,cAAc,KAAK;EAC5B,KAAKJ,IAAI,OAAO,yBAAyB,qBAAqB;EAC9D,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAKe,aACT,oBACA,EAAE,UAAU,KAAKC,KAAK,kBAAkB,EAAE,OAAO,GAAG;GAAC;GAAO,cAAc;EAAmB,CAAC,GAChG;EACA,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,KAAKhB,IAAI,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,KAC1D;EAEA,IAAI,WAKF,UAAe,QAAQ;CAE3B;;;;;;;;;;;;;;;;;;CAmBA,MAAM,yCAIH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,QAAQ,eAAe,eAC/C,MAAM,KAAKe,aACT,qBACA,MACE,KAAKV,MACL,QAAO;GACL,GAA8B;sCACJ,KAAKW,KAAK,kBAAkB;GAKtD,GAAG;;;;;;;;;iBASE,KAAKA,KAAK,aAAa,EAAE;sBACpB,KAAKA,KAAK,eAAe,EAAE;;;;GAWrC,GAA0B;oDACc,KAAKA,KAAK,eAAe,EAAE;;GAGnE,GAAqB;8DAC6B,KAAKA,KAAK,aAAa,EAAE;;EAE7E,GACA,EAAC,MAAM,SAAa,CACtB,CACF;EAEF,OAAO;GACL;GACA,kBAAkB,MAAQ,QAAQ,sBAAsB;GACxD,SAAS;IACP,eAAe,CAAC,GAAG,aAAa;IAChC,aAAa,CAAC,GAAG,WAAW;GAC9B;EACF;CACF;CAEA,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,KAAKD,aAClC,qBACA,KAAKV,GAA6C;qDACH,KAAKW,KAAK,WAAW,GACtE;EACA,OAAO;CACT;;;;;;CAOA,MAAM,mBAGH;EACD,MAAM,CAAC,UAAU,MAAM,KAAKD,aAC1B,sBACA,KAAKV,GAAoE;;sCAEzC,KAAKW,KAAK,WAAW,EAAE;uCACtB,KAAKA,KAAK,kBAAkB,EAAE,qBACjE;EACA,OAAO;CACT;;;;;CAMA,MAAM,qBACJ,gBACA,kBACA,OACmB;EAQnB,QAAO,MAPY,KAAKX,GAAmC;8BACjC,KAAKW,KAAK,WAAW,EAAE;;2BAE1B,eAAe;4BACd,iBAAiB;;eAE9B,SACC,KAAK,EAAC,gBAAe,SAAS;CAC5C;;;;;;;;CASA,iBACE,gBACA,kBACA,YAAY,KACqB;EACjC,OACE,OAAO,cAAc,SAAS,KAAK,YAAY,GAC/C,gEACF;EACA,OAAO,KAAKX,GAA8B;kEACoB,KAAKW,KAAK,WAAW,EAAE;2BAC9D,eAAe;4BACd,iBAAiB;gCACb,OAAO,SAAS;CAG9C;CAMA,mBAAmB,WAAoC;EACrD,OAAO,MAAM,KAAKX,KAAK,OAAM,QAAO;GAGlC,MAAM,GAA0B;kCACJ,KAAKW,KAAK,WAAW,EAAE;;;;GASnD,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;;;;;;;;iBASvC,KAAKA,KAAK,WAAW,EAAE;;wBAEhB,KAAKA,KAAK,WAAW,EAAE,qBAAqB,UAAU;;;;GAKxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;8BACvB,KAAKA,KAAK,kBAAkB,EAAE;GACtD,IAAI,UAAU,KAAKd,SACjB,MAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,OAClF;GAEF,OAAO,OAAO,OAAO;EACvB,CAAC;CACH;;;;CAKA,MAAM,WAAmB,MAAwB;EAI/C,MAAM,OAAO,0BAA0B,IAAI;EAC3C,KAAKe,2BAA2B,KAAK;EAErC,MAAM,SAAS,KAAK;EACpB,KAAKR,OAAO,QAAQ;GAClB;GACA;GACA;GACA,aAAa,MAAM,IAAI,OAAO;EAChC,CAAC;EAED,OAAO;CACT;CAEA,QAAQ;EACN,KAAKA,OAAO,QAAQ,CAAC,OAAO,CAAC;CAC/B;CAEA,QAAQ,YAAwB,MAAsB;EACpD,KAAKA,OAAO,QAAQ,CAAC,cAAc;GAAC;GAAY;EAAI,CAAC,CAAC;CACxD;CAEA,gBAAuC;CAEvC,eAA0C;EACxC,IAAI,CAAC,KAAKS,UACR;EAEF,IACE,KAAKC,kBAAkB,QACvB,KAAKF,0BAA0B,KAAKP,6BACpC;GACA,KAAKV,IAAI,OACP,+BAA+B,KAAKS,OAAO,KAAK,EAAE,qBAAqB,KAAKQ,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,2SAW9H;GACA,KAAKE,gBAAgB,SAAS;EAChC;EACA,OAAO,KAAKA,eAAe;CAC7B;CAEA,4BAA4B;EAC1B,IAAI,KAAKA,kBAAkB;OAGvB,KAAKF,0BACL,KAAKP,8BAA8B,IACnC;IACA,KAAKV,IAAI,OACP,gCAAgC,KAAKS,OAAO,KAAK,EAAE,qBAAqB,KAAKQ,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,KAC/H;IACA,KAAKE,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;;CAEJ;;;;;;;;;;;;CAaA,gBAAgB,IAAsD;EACpE,MAAM,EAAC,UAAS;EAChB,IAAI,MAAM,WAAW,GACnB,OAAO,GAAG;EAEZ,GAAG,QAAQ,CAAC;EACZ,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG;sBACa,KAAKH,KAAK,WAAW,EAAE;;mCAEV,MAAM;;;;;YAMrC,CAAC;EACD,OAAO,GAAG;CACZ;CAEA,WAAW;;;;;CAMX,MAAM,MAAM;EACV,OAAO,CAAC,KAAKE,UAAU,2BAA2B;EAElD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,SAAS;EAC5D,KAAKA,WAAW;EAChB,KAAKE,WAAW;EAEhB,KAAKpB,IAAI,OAAO,iBAAiB;EACjC,KAAKc,iBAAiB,MAAM;EAC5B,IAAI;EACJ,IAAI;GACF,MAAM,KAAKO,cAAc;EAC3B,SAAS,GAAG;GACV,MAAM;GACN,MAAM;EACR,UAAU;GAER,IAAI,KAAKF,kBAAkB,MAAM;IAC/B,KAAKA,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;GACA,KAAKG,oBACH,KAAKb,OAAO,MAAM,EAAE,QAAO,UAAS,UAAU,KAAA,CAAS,GACvD,GACF;GACA,KAAKS,WAAW;GAChB,cAAc;GACd,KAAKlB,IAAI,OAAO,gBAAgB;EAClC;CACF;CAEA,oBAAoB,OAAqB,GAAY;EACnD,IAAI,MAAM,WAAW,GACnB;EAEF,KAAKA,IAAI,OACP,aAAa,MAAM,OAAO,kCAC5B;EACA,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,sBAAsB;EAC1E,KAAK,MAAM,SAAS,OAAO;GACzB,IAAI,UAAU,QACZ;GAGF,QADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;KAC3B,KAAKA,IAAI,OAAO,iBAAiB,WAAW,IAAI;KAChD,WAAW,KAAK,GAAG;KACnB;IACF;GACF;EACF;CACF;CAEA,MAAMqB,gBAAgB;EACpB,IAAI,KAAgC;EACpC,IAAI;EAIJ,IAAI;EACJ,MAAM,cAAc,YAAY;GAC9B,eAAe;GACf,eAAe,KAAA;GACf,MAAM,QAAQ,MAAM,KAAKZ,OAAO,QAAQ;GACxC,IAAI,UAAU,QACZ,eAAe,KAAKK,iBAAiB,UAAU;IAC7C,MAAM;IAGN,MAAM,MAAM;GACd,CAAC;GAEH,OAAO;EACT;EAEA,MAAM,eAAoC,CAAC;EAC3C,IAAI;GACF,QAAQ,MAAM,MAAM,YAAY,OAAO,QAAQ;IAC7C,MAAM,CAAC,WAAW;IAClB,QAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;MACxB,YAAY;MACZ;KACF;KACA,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;MACvB,IAAI,IACF,aAAa,KAAK,UAAU;WAE5B,MAAM,KAAKS,cAAc,CAAC,UAAU,CAAC;MAEvC;KACF;KACA,KAAK;MACH,IAAI,IAAI;OACN,GAAG,KAAK,MAAM;OACd,MAAM,GAAG,KAAK,KAAK;OACnB,KAAK;MACP;MACA;IAEJ;IAEA,MAAM,CAAC,GAAG,WAAW,MAAM,UAAU;IACrC,MAAM,MAAM,QAAQ;IACpB,KAAKN,2BAA2B,KAAK;IAErC,IAAI,QAAQ,SAAS;KACnB,OAAO,CAAC,IAAI,+CAA+C;KAC3D,MAAM,EAAC,SAAS,SAAS,WAAU,SAA2B;KAC9D,QAAa,YAAY,CAAC,CAAC;KAC3B,KAAK;MACH,MAAM,IAAI,gBACR,KAAKjB,IAAI,YAAY,aAAa,SAAS,GAC3C,EAAC,MAAM,eAAmB,CAC5B;MACA,oBAAoB;MACpB,KAAK;MACL,0BAA0B;MAC1B,KAAK,CAAC,OAAO;MACb,OAAO,CAAC;MACR,WAAW,KAAA;KACb;KACA,GAAG,KAAK,IAAI,KAAKK,GAAG;KAGpB,GAAQ,KAAK,SAAQ,OAAM;MACzB,EAA+B;gCACX,KAAKW,KAAK,kBAAkB,EAAE,aAAa,MAC5D,CAAC,YAAY,QAAQ,MAAM,GAC5B,MACF;MACA,OAAO,CAAC;KACV,CAAC;IACH,OAAO;KACL,OAAO,UAAU,2CAA2C,MAAM;KAClE,GAAG;IACL;IAEA,MAAM,QAAsB;KAC1B,WAAW,QAAQ,WAAW,YAAY,GAAG;KAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;KACtD,KAAK,GAAG;KAGR,QAAQ,uBAAuB,MAAM,GAAG;IAC1C;IAEA,IAAI,WAAW,QAAQ,eAAe,MAAM,GAAG;KAM7C,MAAM,KAAKQ,gBAAgB,EAAE;KAC7B,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG,eAAe,KAAKR,KAAK,WAAW,EAAE,GAAG,IAAI,KAAK,KACrD,GAAG,KAAKS,uBAAuB,KAAK,MAAM,CAC5C,CAAC;IACH,OAAO;KAML,GAAG,MAAM,KAAK,KAAK;KACnB,IAAI,GAAG,MAAM,UAAU,KAAKb,qBAAqB;MAK/C,MAAM,YAAY,GAAG;MACrB,KAAUY,gBAAgB,EAAE;MAC5B,MAAM;KACR;IACF;IACA,KAAKE,0BAA0B;IAE/B,IAAI,QAAQ,UAAU;KAIpB,KAAUF,gBAAgB,EAAE;KAE5B,MAAM,EAAC,UAAS,MAAM,GAAG;KACzB,IAAI,UAAU,KAAKtB,SAEjB,GAAG,KAAK,KACN,IAAI,WACF,2CAA2C,OAC7C,CACF;UACK;MAEL,MAAM,gBAAgB;MACtB,GAAQ,KAAK,SAAQ,OAAM,CACzB,EAAE;qBACK,KAAKc,KAAK,kBAAkB,EAAE,OAAO,GAAG,EAAC,cAAa,CAAC,GAChE,CAAC;MACD,GAAG,KAAK,QAAQ;KAClB;KAEA,MAAM,GAAG,KAAK,KAAK;KAGnB,IAAI,GAAG,KACL,KAAKT,aAAa;MAAC;MAAU;MAAQ,EAAC,UAAS;KAAC,CAAC;KAEnD,KAAK;KAIL,MAAM,KAAKgB,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,QAAQ,YAAY;KAG7B,GAAG,KAAK,MAAM;KACd,MAAM,GAAG,KAAK,KAAK;KACnB,KAAK;KAEL,MAAM,KAAKA,cAAc,aAAa,OAAO,CAAC,CAAC;IACjD;GACF;EACF,SAAS,GAAG;GACV,aAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GACzD,MAAM;EACR;CACF;CAEA,MAAMA,cAAc,MAA2B;EAC7C,IAAI,KAAK,WAAW,GAClB;EAGF,MAAM,KAAK,KAAKvB,IAAI,YAAY,QAAQ,SAAS;EAIjD,MAAM,SAAS,IAAI,gBAAgB,IAAI,EAAC,MAAM,SAAa,CAAC;EAC5D,OAAO,IAAI,KAAKK,GAAG;EAEnB,IAAI;EACJ,MAAM,qBAAqB,KAAKS,iBAAiB,UAAU;GACzD,MAAM;GACN,aAAa,KAAK,KAAK,EAAC,YAAY,QAAO,EAAE,EAAE;EACjD,CAAC;EACD,IAAI;GAKF,CAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAA8B;sCACP,KAAKE,KAAK,kBAAkB,EAAE;OAE9D;EACF,SAAS,GAAG;GACV,KAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GAC7C,MAAM;EACR,UAAU;GACR,mBAAmB;EACrB;EAIA,QAAa,IACX,KAAK,KAAI,QACP,KAAKW,SACH,GAAG,YAAY,cAAc,IAAI,WAAW,EAAE,GAC9C,KACA,eACA,MACF,CACF,CACF,EAAE,cAAc,OAAO,QAAQ,CAAC;CAClC;CAEA,MAAMA,SACJ,IACA,EAAC,YAAY,KAAK,QAClB,eACA,QACA;EACA,IAAI,kBAAkB;EACtB,IAAI,oBAAoB,KAAKb,iBAAiB,UAAU;GACtD,MAAM;GACN,YAAY,IAAI;GAChB;EACF,CAAC;EACD,IAAI;GACF,GAAG,OAAO,kBAAkB;GAC5B,MAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,GAAG,OAAO,aAAa;IACvB,MAAM,QAAQ,KAAK,IAAI;IAIvB,IAAI,iBAAiB,IAAI,cAAc,KAAKR;IAC5C,IAAI,QAAQ;IACZ,IAAI;IAEJ,WAAW,MAAM,WAAW,EAA6B;sEACK,KAAKU,KAAK,WAAW,EAAE;gCAC7D,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,GAAI,GAAG;KACxC,kBAAkB;KAClB,mBAAmB,QAAQ;KAE3B,IAAI;MAOF,MAAM,QAAQ,YAAY,IAAI;MAC9B,MAAM;MACN,MAAM,UAAU,YAAY,IAAI,IAAI;MACpC,IAAI,mBACF,GAAG,UAAU,MAAM,SAAS,WAC1B,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,IAAI,GAAG,0CAChD;MAGF,KAAK,MAAM,SAAS,SAClB,IAAI,MAAM,cAAc,IAAI,WAG1B,iBAAiB;WACZ,IAAI,gBAAgB;OACzB,oBAAoB,IAAI,QACtB,6BAA6B,KAAK,CACpC;OACA;MACF,OAAO,IAAI,SAAS,UAClB,MAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,EACvF;WACK;OACL,GAAG,OACD,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,EAC7F;OACA,IAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,EACjF;OACA;MACF;KAEJ,UAAU;MAER,oBAAoB,KAAKF,iBAAiB,UAAU;OAClD,MAAM;OACN,YAAY,IAAI;OAChB;MACF,CAAC;KACH;IACF;IACA,kBAAkB;IAElB,IAAI,gBAAgB;KAClB,MAAM;KACN,GAAG,OACD,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,IAAI,IAAI,MACd,KACH;IACF,OAWE,GAAG,OACD,cAAc,IAAI,GAAG,gBAAgB,IAAI,UAAU,4CACjB,cAAc,wCAElD;IAMF,IAAS,YAAY;GACvB,CAAC;EACH,SAAS,KAAK;GACZ,GAAG,QAAQ,sCAAsC,IAAI,MAAM,GAAG;GAC9D,IAAI,eAAe,iBAAiB;IAClC,MAAM,kBAAkB,KAAKT,KAAK,KAAKJ,MAAM;IAC7C,KAAKO,SAAS,GAAG;GACnB;GACA,IAAI,KAAK,GAAG;EACd,UAAU;GACR,kBAAkB;EACpB;CACF;;;;;;;;;;;;CAaA,uBAAuB,KAA0B,QAAsB;EACrE,OAAO,UAAU,MAAM,EAAE,SAAQ,OAAM,KAAKoB,aAAa,KAAK,EAAE,CAAC;CACnE;CAEA,aAAa,KAA0B,IAAqC;EAC1E,QAAQ,GAAG,IAAX;GACE,KAAK,mBACH,OAAO,CAAC,KAAKC,yBAAyB,KAAK,GAAG,OAAO,GAAG,QAAQ,CAAC;GAEnE,KAAK,mBACH,OAAO,CACL,KAAKC,0BAA0B,KAAK,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CACtE;GAEF,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,GAAG,IAAI;KAAQ,OAAO,GAAG,IAAI;IAAI;IACtD,OAAO,CACL,GAAG,UAAU,KAAKd,KAAK,eAAe,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,MAC3D;GACF;GAEA,KAAK,cAAc;IACjB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,eAAe,EAAE;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MACnD;GACF;GAEA,KAAK,iBAAiB;IACpB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,kBAAkB,GAAG,IAAI;mCACtC,OAAO,iBAAiB,MAAM,kBAAkB,GAAG,KAC9E;GACF;GAEA,KAAK,eAAe;IAClB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,GAAG,QAC7E;GACF;GAEA,KAAK,qBAAqB;IACxB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAC/F;GACF;GAEA,SACE,YAAY,EAAE;EAClB;CACF;CAEA,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;EAAQ;EACtD,OAAO,GAAG;sBACQ,KAAKA,KAAK,eAAe,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEnC,IAAI,GAAG,EAAE;;CAEjC;CAEA,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;EAAQ;EAC/D,OAAO,GAAG;sBACQ,KAAKA,KAAK,aAAa,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEjC,IAAI,GAAG,EAAE;;CAEjC;;;;;CAMA,MAAM,eAAe;EACnB,IAAI,KAAKE,UAAU;GACjB,MAAM,EAAC,SAAS,YAAW,SAAS;GACpC,KAAKT,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;GACtC,MAAM;EACR;CACF;;;;;;CAOA,MAAM,OAAO;EACX,IAAI,KAAKS,UAAU;GACjB,KAAKJ,iBAAiB,KAAK;GAC3B,KAAKd,IAAI,OAAO,YAAY,KAAKS,OAAO,KAAK,EAAE,mBAAmB;GAClE,KAAK,MAAM;GACX,KAAKA,OAAO,QAAQ,MAAM;EAC5B;EACA,IACG,MAAM,UAAU,KAAKW,UAAU,KAAKP,eAAe,MAAO,aAE3D,MAAM,IAAI,WACR,kCAAkC,KAAKA,gBAAgB,GACzD;CAEJ;AACF;AAEA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,gBACA,WACA;EACA,KAAKb,MAAM;EACX,KAAK+B,MAAM;EACX,KAAK,iBAAiB;EACtB,KAAK,eAAe;CACtB;CAEA,YAAY;CAEZ,MAAM,UAAU;EACd,IAAI,KAAKC,WACP;EAEF,KAAKA,YAAY;EACjB,KAAKD,IAAI,QAAQ;EACjB,MAAM,KAAKA,IACR,KAAK,EACL,OAAM,MAAK,KAAK/B,IAAI,OAAO,iCAAiC,CAAC,CAAC;EACjE,KAAKA,IAAI,OAAO,0BAA0B,KAAK,cAAc;CAC/D;AACF;AAEA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB,IAAgB;EAC1D,KAAKA,MAAM,GAAG,YAAY,aAAa,cAAc;EACrD,KAAKC,SAAS;EACd,KAAKI,MAAM;CACb;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKA,IAAI,GAAG,UAAU,KAAKJ,MAAM,EAAE,GAAG,OAAO;CACtD;CAEA,MAAM,UAAU;EACd,MAAM,KAAK,IAAI,gBAAgB,KAAKD,KAAK,EAAC,MAAM,eAAmB,CAAC,EAAE,IACpE,KAAKK,GACP;EACA,MAAM,MAAM,MAAM,GAAG,iBACnB,QAAO,GAA0B;8BACT,KAAKW,KAAK,WAAW,EAAE;;;KAIjD;EACA,IAAI,IAAI,WAAW,GAAG;GACpB,KAAKhB,IAAI,OAAO,4CAA4C;GAC5D,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,MAAM,CAAC,EAAC,eAAc;EACtB,MAAM,CAAC,EAAC,oBAAmB,MAAM,GAAG,iBAClC,QAAO,GAA+B;uCACL,KAAKgB,KAAK,mBAAmB,EAAE;OAElE;EACA,KAAKhB,IAAI,OACP,oBAAoB,UAAU,kCAAkC,gBAClE;EACA,OAAO,IAAI,UAAU,KAAKA,KAAK,IAAI,gBAAgB,SAAS;CAC9D;AACF;;;;;;;;;;;;;;;;;;AA0BA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA,+BAAwB,IAAI,IAAmB;CAE/C;CAEA,YACE,IACA,oBACA,WACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,yBAAyB;EAChE,KAAKiC,sBAAsB;EAC3B,KAAKC,aAAa;CACpB;CAEA,QAAQ;EACN,cAAc,KAAKE,MAAM;EACzB,KAAKA,SAAS,kBACN,KAAK,kCAAkB,IAAI,KAAK,CAAC,GACvC,KAAKH,mBACP,EAAE,MAAM;CACV;CAGA,kBAAkB,KAAW;EAC3B,KAAK,MAAM,CAAC,MAAM,eAAe,KAAKE,aAAa,QAAQ,GACzD,IAAI,IAAI,QAAQ,IAAI,WAAW,QAAQ,KAAK,KAAKF,qBAAqB;GACpE,KAAKjC,IAAI,QACP,kCAAkC,KAAKiC,oBAAoB,SAC3D;IAAC;IAAM;GAAU,CACnB;GACA,KAAKC,2BACH,IAAI,MACF,oCAAoC,KAAKD,oBAAoB,MAAM,WAAW,UAAU,IAAI,GAC9F,CACF;GACA,KAAK,KAAK;GACV;EACF;CAEJ;;;;;;CAOA,UAAU,MAAe,sBAAM,IAAI,KAAK,GAAe;EACrD,KAAKE,aAAa,IAAI,MAAM,GAAG;EAC/B,aAAa;GACX,KAAKA,aAAa,OAAO,IAAI;EAC/B;CACF;CAEA,OAAO;EACL,cAAc,KAAKC,MAAM;EACzB,KAAKD,aAAa,MAAM;CAC1B;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pg.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/pg.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,UAAU,CAAC;AAehC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,QAAQ,EAAE,EAAc,KAAK,YAAY,EAAC,MAAM,UAAU,CAAC;AAClE,OAAO,EAAa,KAAK,SAAS,EAAC,MAAM,oCAAoC,CAAC;AAmB9E,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAmD7D;AAED,iBAAS,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAuBhD;AAcD,wBAAgB,0BAA0B,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAoBvE;AAOD,wBAAgB,0BAA0B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CA8ErE;AAED,iBAAS,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAK/C;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,UAAU,CAAC;AAEvD,MAAM,MAAM,WAAW,GAAG;IACxB;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,GAAI,uBAAoB,WAAgB;;;;;;;;;;;;;;;;;;2BAwBlD,OAAO;;;;;;2BAOP,OAAO;;;;;;2BAQP,MAAM,GAAG,IAAI;;;;;;2BAUb,MAAM;uBACV,MAAM,GAAG,MAAM;;;CAG9B,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,CAAC;CACjB,CAAC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,QAAQ,CAAC,cAAc,CAAC;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,CAAC;CACjB,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"pg.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/pg.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,UAAU,CAAC;AAehC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,QAAQ,EAAE,EAAc,KAAK,YAAY,EAAC,MAAM,UAAU,CAAC;AAClE,OAAO,EAAa,KAAK,SAAS,EAAC,MAAM,oCAAoC,CAAC;AAmB9E,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAmD7D;AAED,iBAAS,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAuBhD;AAcD,wBAAgB,0BAA0B,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAoBvE;AAOD,wBAAgB,0BAA0B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CA8ErE;AAED,iBAAS,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAK/C;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,UAAU,CAAC;AAEvD,MAAM,MAAM,WAAW,GAAG;IACxB;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,GAAI,uBAAoB,WAAgB;;;;;;;;;;;;;;;;;;2BAwBlD,OAAO;;;;;;2BAOP,OAAO;;;;;;2BAQP,MAAM,GAAG,IAAI;;;;;;2BAUb,MAAM;uBACV,MAAM,GAAG,MAAM;;;CAG9B,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,CAAC;CACjB,CAAC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,QAAQ,CAAC,cAAc,CAAC;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,CAAC;CACjB,CAAC,CAAC;AAkBH,KAAK,oBAAoB,GAAG;IAC1B,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;CACvB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,SAAS,SAA+B,IAEhC,SAAS,oBAAoB,KAAG,MAAM,CAkD/C;AAED,wBAAgB,QAAQ,CACtB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EAGrB,eAAe,EAAE,MAAM,EACvB,OAAO,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC;IACzB,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAC7B,IAAI,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;CAC/B,CAAC,EACF,IAAI,CAAC,EAAE,WAAW,GACjB,UAAU,CA6EZ;AAED,wBAAsB,eAAe,CACnC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,QAAQ,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAcrB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,mBAAmB,QAE/D;AAED,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAOhD,CAAC;AAEF,wBAAgB,eAAe,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,KAAK,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,WAE1E;AA8BD,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,OAAO,WAiB/C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pg.js","names":[],"sources":["../../../../../zero-cache/src/types/pg.ts"],"sourcesContent":["import {Socket} from 'node:net';\nimport {\n PG_CONFIGURATION_LIMIT_EXCEEDED,\n PG_CONNECTION_DOES_NOT_EXIST,\n PG_CONNECTION_EXCEPTION,\n PG_CONNECTION_FAILURE,\n PG_INSUFFICIENT_PRIVILEGE,\n PG_INVALID_AUTHORIZATION_SPECIFICATION,\n PG_INVALID_CATALOG_NAME,\n PG_INVALID_PASSWORD,\n PG_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION,\n PG_TOO_MANY_CONNECTIONS,\n} from '@drdgvhbh/postgres-error-codes';\nimport {PreciseDate} from '@google-cloud/precise-date';\nimport {OID} from '@postgresql-typed/oids';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres, {type Notice, type PostgresType} from 'postgres';\nimport {BigIntJSON, type JSONValue} from '../../../shared/src/bigint-json.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {ConfigurationError} from './configuration-error.ts';\nimport {\n DATE,\n JSON,\n JSONB,\n NUMERIC,\n TIME,\n TIMESTAMP,\n TIMESTAMPTZ,\n TIMETZ,\n} from './pg-types.ts';\n\n// Matches: YEAR-MM-DD HH:MM:SS[.fraction][+-TZ[:MM]][ BC]\nconst pgTimestampRe =\n /^(\\d+)-(\\d{2}-\\d{2}) (\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?)([+-]\\d{1,2}(?::\\d{2})?)?(?: BC)?$/;\n\n// exported for testing.\nexport function timestampToFpMillis(timestamp: string): number {\n if (timestamp === 'infinity') return Infinity;\n if (timestamp === '-infinity') return -Infinity;\n const match = timestamp.match(pgTimestampRe);\n if (!match) {\n throw new Error(`Error parsing ${timestamp}`);\n }\n\n const [, yearStr, monthDay, time, tz] = match;\n const bc = timestamp.endsWith(' BC');\n\n let year = Number(yearStr);\n if (bc) {\n // Postgres: 1 BC = JS year 0, 2 BC = JS year -1, N BC = -(N-1)\n year = -(year - 1);\n }\n\n // Format year as ISO 8601 expanded year if needed.\n // https://tc39.es/ecma262/#sec-expanded-years\n let isoYear: string;\n if (year >= 0 && year <= 9999) {\n isoYear = String(year).padStart(4, '0');\n } else if (year >= 0) {\n isoYear = '+' + String(year).padStart(6, '0');\n } else {\n isoYear = '-' + String(Math.abs(year)).padStart(6, '0');\n }\n\n // Build a UTC ISO string so PreciseDate returns microsecond precision.\n const utcString = `${isoYear}-${monthDay}T${time}Z`;\n\n try {\n const fullTime = new PreciseDate(utcString).getFullTime();\n const millis = Number(fullTime / 1_000_000n);\n const nanos = Number(fullTime % 1_000_000n);\n const ret = millis + nanos * 1e-6; // floating point milliseconds\n\n // Add back in the timezone offset. We passed local time as UTC,\n // so a positive offset means we need to subtract, and vice versa.\n if (tz) {\n const positiveOffset = tz.startsWith('+');\n const [hours, minutes] = tz.split(':');\n const offset =\n Math.abs(Number(hours)) * 60 + (minutes ? Number(minutes) : 0);\n const offsetMillis = offset * 60 * 1_000;\n return positiveOffset ? ret - offsetMillis : ret + offsetMillis;\n }\n return ret;\n } catch (e) {\n throw new Error(`Error parsing ${timestamp}`, {cause: e});\n }\n}\n\nfunction serializeTimestamp(val: unknown): string {\n switch (typeof val) {\n case 'string':\n return val; // Let Postgres parse it\n case 'number': {\n if (Number.isInteger(val)) {\n return new PreciseDate(val).toISOString();\n }\n // Convert floating point to bigint nanoseconds.\n const nanoseconds =\n 1_000_000n * BigInt(Math.trunc(val)) +\n BigInt(Math.trunc((val % 1) * 1e6));\n return new PreciseDate(nanoseconds).toISOString();\n }\n // Note: Don't support bigint inputs until we decide what the semantics are (e.g. micros vs nanos)\n // case 'bigint':\n // return new PreciseDate(val).toISOString();\n default:\n if (val instanceof Date) {\n return val.toISOString();\n }\n }\n throw new Error(`Unsupported type \"${typeof val}\" for timestamp: ${val}`);\n}\n\nconst MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;\n\nfunction serializeTime(x: unknown, type: 'time' | 'timetz'): string {\n switch (typeof x) {\n case 'string':\n return x; // Let Postgres parse it\n case 'number':\n return millisecondsToPostgresTime(x);\n }\n throw new Error(`Unsupported type \"${typeof x}\" for ${type}: ${x}`);\n}\n\nexport function millisecondsToPostgresTime(milliseconds: number): string {\n if (milliseconds < 0) {\n throw new Error('Milliseconds cannot be negative');\n }\n\n if (milliseconds >= MILLISECONDS_PER_DAY) {\n throw new Error(\n `Milliseconds cannot exceed 24 hours (${MILLISECONDS_PER_DAY}ms)`,\n );\n }\n\n milliseconds = Math.floor(milliseconds); // Ensure it's an integer\n\n const totalSeconds = Math.floor(milliseconds / 1000);\n const hours = Math.floor(totalSeconds / 3600);\n const minutes = Math.floor((totalSeconds % 3600) / 60);\n const seconds = totalSeconds % 60;\n const ms = milliseconds % 1000;\n\n return `${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}.${ms.toString().padStart(3, '0')}+00`;\n}\n\n// Regular expression to match HH:MM:SS, HH:MM:SS.mmm, or HH:MM:SS+00 / HH:MM:SS.mmm+00\n// Supports optional timezone offset\nconst timeRegex =\n /^(\\d{1,2}):(\\d{2}):(\\d{2})(?:\\.(\\d{1,6}))?(?:([+-])(\\d{1,2})(?::(\\d{2}))?)?$/;\n\nexport function postgresTimeToMilliseconds(timeString: string): number {\n // Validate basic format\n if (!timeString || typeof timeString !== 'string') {\n throw new Error('Invalid time string: must be a non-empty string');\n }\n\n const match = timeString.match(timeRegex);\n\n if (!match) {\n throw new Error(\n `Invalid time format: \"${timeString}\". Expected HH:MM:SS[.mmm][+|-HH[:MM]]`,\n );\n }\n\n // Extract components\n const hours = parseInt(match[1], 10);\n const minutes = parseInt(match[2], 10);\n const seconds = parseInt(match[3], 10);\n // Handle optional milliseconds, pad right with zeros if needed\n let milliseconds = 0;\n if (match[4]) {\n // Pad microseconds to 6 digits\n const msString = match[4].padEnd(6, '0');\n // slice milliseconds out of the microseconds\n // e.g. 123456 -> 123, 1234 -> 123,\n milliseconds = parseInt(msString.slice(0, 3), 10);\n }\n\n // Validate ranges\n if (hours < 0 || hours > 24) {\n throw new Error(\n `Invalid hours: ${hours}. Must be between 0 and 24 (24 means end of day)`,\n );\n }\n\n if (minutes < 0 || minutes >= 60) {\n throw new Error(`Invalid minutes: ${minutes}. Must be between 0 and 59`);\n }\n\n if (seconds < 0 || seconds >= 60) {\n throw new Error(`Invalid seconds: ${seconds}. Must be between 0 and 59`);\n }\n\n if (milliseconds < 0 || milliseconds >= 1000) {\n throw new Error(\n `Invalid milliseconds: ${milliseconds}. Must be between 0 and 999`,\n );\n }\n\n // Special case: PostgreSQL allows 24:00:00 to represent end of day\n if (hours === 24 && (minutes !== 0 || seconds !== 0 || milliseconds !== 0)) {\n throw new Error(\n 'Invalid time: when hours is 24, minutes, seconds, and milliseconds must be 0',\n );\n }\n\n // Calculate total milliseconds\n let totalMs =\n hours * 3600000 + minutes * 60000 + seconds * 1000 + milliseconds;\n\n // Timezone Offset\n if (match[5]) {\n const sign = match[5] === '+' ? 1 : -1;\n const tzHours = parseInt(match[6], 10);\n const tzMinutes = match[7] ? parseInt(match[7], 10) : 0;\n const offsetMs = sign * (tzHours * 3600000 + tzMinutes * 60000);\n totalMs -= offsetMs;\n }\n\n // Normalize to 0-24h only if outside valid range\n if (totalMs > MILLISECONDS_PER_DAY || totalMs < 0) {\n return (\n ((totalMs % MILLISECONDS_PER_DAY) + MILLISECONDS_PER_DAY) %\n MILLISECONDS_PER_DAY\n );\n }\n\n return totalMs;\n}\n\nfunction dateToUTCMidnight(date: string): number {\n if (date === 'infinity') return Infinity;\n if (date === '-infinity') return -Infinity;\n const d = new Date(date);\n return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate());\n}\n\n/**\n * The (javascript) types of objects that can be returned by our configured\n * Postgres clients. For initial-sync, these comes from the postgres.js client:\n *\n * https://github.com/porsager/postgres/blob/master/src/types.js\n *\n * and for the replication stream these come from the the node-postgres client:\n *\n * https://github.com/brianc/node-pg-types/blob/master/lib/textParsers.js\n */\nexport type PostgresValueType = JSONValue | Uint8Array;\n\nexport type TypeOptions = {\n /**\n * Sends strings directly as JSON values (i.e. without JSON stringification).\n * The application is responsible for ensuring that string inputs for JSON\n * columns are already stringified. Other data types (e.g. objects) will\n * still be stringified by the pg client.\n */\n sendStringAsJson?: boolean;\n};\n\n/**\n * Configures types for the Postgres.js client library (`postgres`).\n *\n * @param jsonAsString Keep JSON / JSONB values as strings instead of parsing.\n */\nexport const postgresTypeConfig = ({sendStringAsJson}: TypeOptions = {}) => ({\n // Type the type IDs as `number` so that Typescript doesn't complain about\n // referencing external types during type inference.\n types: {\n bigint: postgres.BigInt,\n json: {\n to: JSON,\n from: [JSON, JSONB],\n serialize: sendStringAsJson\n ? (x: unknown) => (typeof x === 'string' ? x : BigIntJSON.stringify(x))\n : BigIntJSON.stringify,\n parse: BigIntJSON.parse,\n },\n // Timestamps are converted to PreciseDate objects.\n timestamp: {\n to: TIMESTAMP,\n from: [TIMESTAMP, TIMESTAMPTZ],\n serialize: serializeTimestamp,\n parse: timestampToFpMillis,\n },\n // Times are converted as strings\n time: {\n to: TIME,\n from: [TIME, TIMETZ],\n serialize: (x: unknown) => serializeTime(x, 'time'),\n parse: postgresTimeToMilliseconds,\n },\n\n timetz: {\n to: TIMETZ,\n from: [TIME, TIMETZ],\n serialize: (x: unknown) => serializeTime(x, 'timetz'),\n parse: postgresTimeToMilliseconds,\n },\n\n // The DATE type is stored directly as the PG normalized date string.\n date: {\n to: DATE,\n from: [DATE],\n serialize: (x: string | Date) =>\n (x instanceof Date ? x : new Date(x)).toISOString(),\n parse: dateToUTCMidnight,\n },\n // Returns a `js` number which can lose precision for large numbers.\n // JS number is 53 bits so this should generally not occur.\n // An API will be provided for users to override this type.\n numeric: {\n to: NUMERIC,\n from: [NUMERIC],\n serialize: (x: number) => String(x), // pg expects a string\n parse: (x: string | number) => Number(x),\n },\n },\n});\n\nexport type PostgresDB = postgres.Sql<{\n bigint: bigint;\n json: JSONValue;\n}>;\n\nexport type PostgresTransaction = postgres.TransactionSql<{\n bigint: bigint;\n json: JSONValue;\n}>;\n\n// Guards against half-open TCP connections, which otherwise hang the pool\n// indefinitely: a proxy can keep the client-facing socket alive (ACKing TCP\n// keepalives) after its backend is gone, so no 'close' or 'error' event ever\n// fires and statements like BEGIN / COMMIT — which are not covered by the\n// TransactionPool's statementResponseTimeout — await forever.\n// See https://github.com/porsager/postgres/issues/1089.\n//\n// If no bytes are read or written for this long, the socket is reset, which\n// rejects all in-flight queries and lets the pool recover. Wire activity\n// resets the timer, so streaming operations (e.g. COPY) are safe, but\n// statements that legitimately compute silently for longer (e.g. index\n// builds in migrations) must raise this. Settable as an emergency measure\n// (0 disables); deliberately not exposed as a server option.\nconst SOCKET_INACTIVITY_TIMEOUT_MS = parseInt(\n process.env.ZERO_PG_SOCKET_INACTIVITY_TIMEOUT ?? '120000',\n);\n\ntype SocketFactoryOptions = {\n host: string[];\n port: number[];\n path?: string | false;\n};\n\n/**\n * Creates the custom socket factory passed to postgres.js via its (untyped)\n * `socket` option. postgres.js does not call `connect()` on custom sockets,\n * and its TLS upgrade reads `socket.host` for the SNI servername, so the\n * factory must handle both itself.\n *\n * Note: unlike postgres.js's internal socket, reconnects do not rotate\n * through multiple hosts; multi-host URIs are not used by the zero-cache.\n *\n * Exported for testing.\n */\nexport function inactivityTimeoutSocket(\n lc: LogContext,\n timeoutMs = SOCKET_INACTIVITY_TIMEOUT_MS,\n) {\n return (options: SocketFactoryOptions): Socket => {\n const socket = new Socket();\n const target = options.path\n ? options.path\n : `${options.host[0]}:${options.port[0]}`;\n if (timeoutMs > 0) {\n // socket.setTimeout() cannot be used here: it registers its callback\n // as a 'timeout' event listener, and postgres.js removes all listeners\n // from the raw socket when upgrading it to TLS (removeAllListeners()\n // in secure()), which would silently disarm the watchdog on every TLS\n // connection. An interval is not a socket listener, so it survives the\n // upgrade, and encrypted traffic still flows through this raw socket,\n // advancing its byte counters.\n //\n // The counters are sampled once per timeout period, so a half-open\n // socket is reset after one to two timeout periods of inactivity.\n let lastActivity = -1;\n const watchdog = setInterval(() => {\n if (socket.destroyed) {\n clearInterval(watchdog);\n return;\n }\n const activity = socket.bytesRead + socket.bytesWritten;\n if (activity === lastActivity) {\n lc.warn?.(\n `resetting connection to ${target} after ${timeoutMs} ms of inactivity ` +\n `(likely half-open); in-flight queries will be rejected`,\n );\n clearInterval(watchdog);\n socket.resetAndDestroy();\n } else {\n lastActivity = activity;\n }\n }, timeoutMs);\n watchdog.unref();\n // Best-effort cleanup; removed on TLS upgrade, in which case the\n // destroyed check above clears the (unref'ed) interval instead.\n socket.once('close', () => clearInterval(watchdog));\n }\n if (options.path) {\n socket.connect(options.path);\n } else {\n const [host] = options.host;\n const [port] = options.port;\n socket.connect(port, host);\n // Read by postgres.js for the TLS SNI servername.\n Object.assign(socket, {host, port});\n }\n return socket;\n };\n}\n\nexport function pgClient(\n lc: LogContext,\n connectionURI: string,\n // A descriptive name for the code that is making the connection, so that we\n // can debug things like deadlocks.\n applicationName: string,\n options?: postgres.Options<{\n bigint: PostgresType<bigint>;\n json: PostgresType<JSONValue>;\n }>,\n opts?: TypeOptions,\n): PostgresDB {\n applicationName = `zero-${applicationName}`;\n\n // Postgres carries row values in the `detail`, `hint`, `where` and\n // `internal_query` fields -- e.g. a unique violation reports\n // `Key (email)=(someone@example.com) already exists` in `detail`. Forward\n // only the severity, SQLSTATE and schema location; never the Notice itself.\n const safeNotice = (n: Notice) => ({\n severity: n.severity,\n code: n.code,\n message: n.message,\n schema: n.schema_name,\n table: n.table_name,\n column: n.column_name,\n constraint: n.constraint_name,\n routine: n.routine,\n });\n\n const onnotice = (n: Notice) => {\n // https://www.postgresql.org/docs/current/plpgsql-errors-and-messages.html#PLPGSQL-STATEMENTS-RAISE\n switch (n.severity) {\n case 'NOTICE':\n return; // silenced\n case 'DEBUG':\n lc.debug?.('pg notice', safeNotice(n));\n return;\n case 'WARNING':\n lc.warn?.('pg notice', safeNotice(n));\n return;\n case 'EXCEPTION':\n lc.error?.('pg notice', safeNotice(n));\n return;\n case 'LOG':\n case 'INFO':\n default:\n lc.info?.('pg notice', safeNotice(n));\n }\n };\n const url = new URL(connectionURI);\n const sslFlag =\n url.searchParams.get('ssl') ?? url.searchParams.get('sslmode') ?? 'prefer';\n\n let ssl: boolean | 'prefer' | {rejectUnauthorized: boolean};\n if (sslFlag === 'disable' || sslFlag === 'false') {\n ssl = false;\n } else if (sslFlag === 'no-verify') {\n ssl = {rejectUnauthorized: false};\n } else {\n ssl = sslFlag as 'prefer';\n }\n\n // Set connections to expire between 5 and 10 minutes to free up state on PG.\n const maxLifetimeSeconds = randInt(5 * 60, 10 * 60);\n const providedConnection = options?.connection ?? {};\n\n // postgres.js supports a custom socket factory via the `socket` option,\n // but it is missing from its type declarations, hence the untyped spread.\n const socketFactory = {\n socket: inactivityTimeoutSocket(lc.withContext('appName', applicationName)),\n };\n\n return postgres(connectionURI, {\n ...postgresTypeConfig(opts),\n onnotice,\n ['max_lifetime']: maxLifetimeSeconds,\n // Close idle connections cleanly before the socket inactivity timer\n // (which cannot distinguish an idle pool connection from a hung query)\n // resets them.\n ['idle_timeout']: 60,\n ssl,\n ...socketFactory,\n ...options,\n connection: {\n ...providedConnection,\n ['application_name']: applicationName,\n },\n });\n}\n\nexport async function connectPgClient(\n ...args: Parameters<typeof pgClient>\n): Promise<PostgresDB> {\n const db = pgClient(...args);\n try {\n await db`SELECT 1`.simple().execute();\n return db;\n } catch (e) {\n if (isPostgresConfigError(e)) {\n throw new ConfigurationError(\n 'Unable to connect to Postgres. Check your database configuration.',\n {cause: e},\n );\n }\n throw e;\n }\n}\n\n/**\n * Disables any statement_timeout for the current transaction. By default,\n * Postgres does not impose a statement timeout, but some users and providers\n * set one at the database level (even though it is explicitly discouraged by\n * the Postgres documentation).\n *\n * Zero logic in particular often does not fit into the category of general\n * application logic; for potentially long-running operations like migrations\n * and background cleanup, the statement timeout should be disabled to prevent\n * these operations from timing out.\n */\nexport function disableStatementTimeout(sql: PostgresTransaction) {\n void sql`SET LOCAL statement_timeout = 0;`.execute().catch(() => {});\n}\n\nexport const typeNameByOID: Record<number, string> = Object.freeze(\n Object.fromEntries(\n Object.entries(OID).map(([name, oid]) => [\n oid,\n name.startsWith('_') ? `${name.substring(1)}[]` : name,\n ]),\n ),\n);\n\nexport function isPostgresError(e: unknown, ...codes: [string, ...string[]]) {\n return e instanceof postgres.PostgresError && codes.includes(e.code);\n}\n\nfunction isPostgresStartupConnectionError(e: unknown) {\n if (!(e instanceof Error) || !('code' in e)) {\n return false;\n }\n const {code} = e;\n // postgres.js throws plain Error instances for connection failures that\n // happen before Postgres can return a SQLSTATE. These are still startup\n // configuration problems for Zero: wrong host/port, unreachable DB, DNS,\n // TLS certificate mismatch, etc.\n return (\n typeof code === 'string' &&\n [\n 'CONNECT_TIMEOUT',\n 'ECONNREFUSED',\n 'ECONNRESET',\n 'EHOSTUNREACH',\n 'ENETUNREACH',\n 'ENOTFOUND',\n 'EAI_AGAIN',\n 'ETIMEDOUT',\n 'ERR_TLS_CERT_ALTNAME_INVALID',\n 'SELF_SIGNED_CERT_IN_CHAIN',\n 'DEPTH_ZERO_SELF_SIGNED_CERT',\n 'UNABLE_TO_VERIFY_LEAF_SIGNATURE',\n ].includes(code)\n );\n}\n\nexport function isPostgresConfigError(e: unknown) {\n return (\n isPostgresStartupConnectionError(e) ||\n isPostgresError(\n e,\n PG_CONNECTION_EXCEPTION,\n PG_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION,\n PG_CONNECTION_DOES_NOT_EXIST,\n PG_CONNECTION_FAILURE,\n PG_INVALID_AUTHORIZATION_SPECIFICATION,\n PG_INVALID_PASSWORD,\n PG_INVALID_CATALOG_NAME,\n PG_INSUFFICIENT_PRIVILEGE,\n PG_TOO_MANY_CONNECTIONS,\n PG_CONFIGURATION_LIMIT_EXCEEDED,\n )\n );\n}\n"],"mappings":";;;;;;;;;;AAgCA,IAAM,gBACJ;AAGF,SAAgB,oBAAoB,WAA2B;CAC7D,IAAI,cAAc,YAAY,OAAO;CACrC,IAAI,cAAc,aAAa,OAAO;CACtC,MAAM,QAAQ,UAAU,MAAM,aAAa;CAC3C,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,WAAW;CAG9C,MAAM,GAAG,SAAS,UAAU,MAAM,MAAM;CACxC,MAAM,KAAK,UAAU,SAAS,KAAK;CAEnC,IAAI,OAAO,OAAO,OAAO;CACzB,IAAI,IAEF,OAAO,EAAE,OAAO;CAKlB,IAAI;CACJ,IAAI,QAAQ,KAAK,QAAQ,MACvB,UAAU,OAAO,IAAI,EAAE,SAAS,GAAG,GAAG;MACjC,IAAI,QAAQ,GACjB,UAAU,MAAM,OAAO,IAAI,EAAE,SAAS,GAAG,GAAG;MAE5C,UAAU,MAAM,OAAO,KAAK,IAAI,IAAI,CAAC,EAAE,SAAS,GAAG,GAAG;CAIxD,MAAM,YAAY,GAAG,QAAQ,GAAG,SAAS,GAAG,KAAK;CAEjD,IAAI;EACF,MAAM,WAAW,IAAI,YAAY,SAAS,EAAE,YAAY;EAGxD,MAAM,MAFS,OAAO,WAAW,QAErB,IADE,OAAO,WAAW,QACX,IAAQ;EAI7B,IAAI,IAAI;GACN,MAAM,iBAAiB,GAAG,WAAW,GAAG;GACxC,MAAM,CAAC,OAAO,WAAW,GAAG,MAAM,GAAG;GAGrC,MAAM,gBADJ,KAAK,IAAI,OAAO,KAAK,CAAC,IAAI,MAAM,UAAU,OAAO,OAAO,IAAI,MAChC,KAAK;GACnC,OAAO,iBAAiB,MAAM,eAAe,MAAM;EACrD;EACA,OAAO;CACT,SAAS,GAAG;EACV,MAAM,IAAI,MAAM,iBAAiB,aAAa,EAAC,OAAO,EAAC,CAAC;CAC1D;AACF;AAEA,SAAS,mBAAmB,KAAsB;CAChD,QAAQ,OAAO,KAAf;EACE,KAAK,UACH,OAAO;EACT,KAAK;GACH,IAAI,OAAO,UAAU,GAAG,GACtB,OAAO,IAAI,YAAY,GAAG,EAAE,YAAY;GAM1C,OAAO,IAAI,YAFT,WAAa,OAAO,KAAK,MAAM,GAAG,CAAC,IACnC,OAAO,KAAK,MAAO,MAAM,IAAK,GAAG,CAAC,CACF,EAAE,YAAY;EAKlD,SACE,IAAI,eAAe,MACjB,OAAO,IAAI,YAAY;CAE7B;CACA,MAAM,IAAI,MAAM,qBAAqB,OAAO,IAAI,mBAAmB,KAAK;AAC1E;AAEA,IAAM,uBAAuB,OAAU,KAAK;AAE5C,SAAS,cAAc,GAAY,MAAiC;CAClE,QAAQ,OAAO,GAAf;EACE,KAAK,UACH,OAAO;EACT,KAAK,UACH,OAAO,2BAA2B,CAAC;CACvC;CACA,MAAM,IAAI,MAAM,qBAAqB,OAAO,EAAE,QAAQ,KAAK,IAAI,GAAG;AACpE;AAEA,SAAgB,2BAA2B,cAA8B;CACvE,IAAI,eAAe,GACjB,MAAM,IAAI,MAAM,iCAAiC;CAGnD,IAAI,gBAAgB,sBAClB,MAAM,IAAI,MACR,wCAAwC,qBAAqB,IAC/D;CAGF,eAAe,KAAK,MAAM,YAAY;CAEtC,MAAM,eAAe,KAAK,MAAM,eAAe,GAAI;CACnD,MAAM,QAAQ,KAAK,MAAM,eAAe,IAAI;CAC5C,MAAM,UAAU,KAAK,MAAO,eAAe,OAAQ,EAAE;CACrD,MAAM,UAAU,eAAe;CAC/B,MAAM,KAAK,eAAe;CAE1B,OAAO,GAAG,MAAM,SAAS,EAAE,SAAS,GAAG,GAAG,EAAE,GAAG,QAAQ,SAAS,EAAE,SAAS,GAAG,GAAG,EAAE,GAAG,QAAQ,SAAS,EAAE,SAAS,GAAG,GAAG,EAAE,GAAG,GAAG,SAAS,EAAE,SAAS,GAAG,GAAG,EAAE;AAC9J;AAIA,IAAM,YACJ;AAEF,SAAgB,2BAA2B,YAA4B;CAErE,IAAI,CAAC,cAAc,OAAO,eAAe,UACvC,MAAM,IAAI,MAAM,iDAAiD;CAGnE,MAAM,QAAQ,WAAW,MAAM,SAAS;CAExC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,yBAAyB,WAAW,uCACtC;CAIF,MAAM,QAAQ,SAAS,MAAM,IAAI,EAAE;CACnC,MAAM,UAAU,SAAS,MAAM,IAAI,EAAE;CACrC,MAAM,UAAU,SAAS,MAAM,IAAI,EAAE;CAErC,IAAI,eAAe;CACnB,IAAI,MAAM,IAAI;EAEZ,MAAM,WAAW,MAAM,GAAG,OAAO,GAAG,GAAG;EAGvC,eAAe,SAAS,SAAS,MAAM,GAAG,CAAC,GAAG,EAAE;CAClD;CAGA,IAAI,QAAQ,KAAK,QAAQ,IACvB,MAAM,IAAI,MACR,kBAAkB,MAAM,iDAC1B;CAGF,IAAI,UAAU,KAAK,WAAW,IAC5B,MAAM,IAAI,MAAM,oBAAoB,QAAQ,2BAA2B;CAGzE,IAAI,UAAU,KAAK,WAAW,IAC5B,MAAM,IAAI,MAAM,oBAAoB,QAAQ,2BAA2B;CAGzE,IAAI,eAAe,KAAK,gBAAgB,KACtC,MAAM,IAAI,MACR,yBAAyB,aAAa,4BACxC;CAIF,IAAI,UAAU,OAAO,YAAY,KAAK,YAAY,KAAK,iBAAiB,IACtE,MAAM,IAAI,MACR,8EACF;CAIF,IAAI,UACF,QAAQ,OAAU,UAAU,MAAQ,UAAU,MAAO;CAGvD,IAAI,MAAM,IAAI;EACZ,MAAM,OAAO,MAAM,OAAO,MAAM,IAAI;EACpC,MAAM,UAAU,SAAS,MAAM,IAAI,EAAE;EACrC,MAAM,YAAY,MAAM,KAAK,SAAS,MAAM,IAAI,EAAE,IAAI;EACtD,MAAM,WAAW,QAAQ,UAAU,OAAU,YAAY;EACzD,WAAW;CACb;CAGA,IAAI,UAAU,wBAAwB,UAAU,GAC9C,QACI,UAAU,uBAAwB,wBACpC;CAIJ,OAAO;AACT;AAEA,SAAS,kBAAkB,MAAsB;CAC/C,IAAI,SAAS,YAAY,OAAO;CAChC,IAAI,SAAS,aAAa,OAAO;CACjC,MAAM,IAAI,IAAI,KAAK,IAAI;CACvB,OAAO,KAAK,IAAI,EAAE,eAAe,GAAG,EAAE,YAAY,GAAG,EAAE,WAAW,CAAC;AACrE;;;;;;AA6BA,IAAa,sBAAsB,EAAC,qBAAiC,CAAC,OAAO,EAG3E,OAAO;CACL,QAAQ,SAAS;CACjB,MAAM;EACJ,IAAA;EACA,MAAM,CAAA,KAAO,KAAK;EAClB,WAAW,oBACN,MAAgB,OAAO,MAAM,WAAW,IAAI,WAAW,UAAU,CAAC,IACnE,WAAW;EACf,OAAO,WAAW;CACpB;CAEA,WAAW;EACT,IAAI;EACJ,MAAM,CAAC,WAAW,WAAW;EAC7B,WAAW;EACX,OAAO;CACT;CAEA,MAAM;EACJ,IAAI;EACJ,MAAM,CAAC,MAAM,MAAM;EACnB,YAAY,MAAe,cAAc,GAAG,MAAM;EAClD,OAAO;CACT;CAEA,QAAQ;EACN,IAAI;EACJ,MAAM,CAAC,MAAM,MAAM;EACnB,YAAY,MAAe,cAAc,GAAG,QAAQ;EACpD,OAAO;CACT;CAGA,MAAM;EACJ,IAAI;EACJ,MAAM,CAAC,IAAI;EACX,YAAY,OACT,aAAa,OAAO,IAAI,IAAI,KAAK,CAAC,GAAG,YAAY;EACpD,OAAO;CACT;CAIA,SAAS;EACP,IAAI;EACJ,MAAM,CAAC,OAAO;EACd,YAAY,MAAc,OAAO,CAAC;EAClC,QAAQ,MAAuB,OAAO,CAAC;CACzC;AACF,EACF;AAyBA,IAAM,+BAA+B,SACnC,QAAQ,IAAI,qCAAqC,QACnD;;;;;;;;;;;;AAmBA,SAAgB,wBACd,IACA,YAAY,8BACZ;CACA,QAAQ,YAA0C;EAChD,MAAM,SAAS,IAAI,OAAO;EAC1B,MAAM,SAAS,QAAQ,OACnB,QAAQ,OACR,GAAG,QAAQ,KAAK,GAAG,GAAG,QAAQ,KAAK;EACvC,IAAI,YAAY,GAAG;GAWjB,IAAI,eAAe;GACnB,MAAM,WAAW,kBAAkB;IACjC,IAAI,OAAO,WAAW;KACpB,cAAc,QAAQ;KACtB;IACF;IACA,MAAM,WAAW,OAAO,YAAY,OAAO;IAC3C,IAAI,aAAa,cAAc;KAC7B,GAAG,OACD,2BAA2B,OAAO,SAAS,UAAU,yEAEvD;KACA,cAAc,QAAQ;KACtB,OAAO,gBAAgB;IACzB,OACE,eAAe;GAEnB,GAAG,SAAS;GACZ,SAAS,MAAM;GAGf,OAAO,KAAK,eAAe,cAAc,QAAQ,CAAC;EACpD;EACA,IAAI,QAAQ,MACV,OAAO,QAAQ,QAAQ,IAAI;OACtB;GACL,MAAM,CAAC,QAAQ,QAAQ;GACvB,MAAM,CAAC,QAAQ,QAAQ;GACvB,OAAO,QAAQ,MAAM,IAAI;GAEzB,OAAO,OAAO,QAAQ;IAAC;IAAM;GAAI,CAAC;EACpC;EACA,OAAO;CACT;AACF;AAEA,SAAgB,SACd,IACA,eAGA,iBACA,SAIA,MACY;CACZ,kBAAkB,QAAQ;CAM1B,MAAM,cAAc,OAAe;EACjC,UAAU,EAAE;EACZ,MAAM,EAAE;EACR,SAAS,EAAE;EACX,QAAQ,EAAE;EACV,OAAO,EAAE;EACT,QAAQ,EAAE;EACV,YAAY,EAAE;EACd,SAAS,EAAE;CACb;CAEA,MAAM,YAAY,MAAc;EAE9B,QAAQ,EAAE,UAAV;GACE,KAAK,UACH;GACF,KAAK;IACH,GAAG,QAAQ,aAAa,WAAW,CAAC,CAAC;IACrC;GACF,KAAK;IACH,GAAG,OAAO,aAAa,WAAW,CAAC,CAAC;IACpC;GACF,KAAK;IACH,GAAG,QAAQ,aAAa,WAAW,CAAC,CAAC;IACrC;GAGF,SACE,GAAG,OAAO,aAAa,WAAW,CAAC,CAAC;EACxC;CACF;CACA,MAAM,MAAM,IAAI,IAAI,aAAa;CACjC,MAAM,UACJ,IAAI,aAAa,IAAI,KAAK,KAAK,IAAI,aAAa,IAAI,SAAS,KAAK;CAEpE,IAAI;CACJ,IAAI,YAAY,aAAa,YAAY,SACvC,MAAM;MACD,IAAI,YAAY,aACrB,MAAM,EAAC,oBAAoB,MAAK;MAEhC,MAAM;CAIR,MAAM,qBAAqB,QAAQ,KAAQ,GAAO;CAClD,MAAM,qBAAqB,SAAS,cAAc,CAAC;CAInD,MAAM,gBAAgB,EACpB,QAAQ,wBAAwB,GAAG,YAAY,WAAW,eAAe,CAAC,EAC5E;CAEA,OAAO,SAAS,eAAe;EAC7B,GAAG,mBAAmB,IAAI;EAC1B;GACC,iBAAiB;GAIjB,iBAAiB;EAClB;EACA,GAAG;EACH,GAAG;EACH,YAAY;GACV,GAAG;IACF,qBAAqB;EACxB;CACF,CAAC;AACH;AAEA,eAAsB,gBACpB,GAAG,MACkB;CACrB,MAAM,KAAK,SAAS,GAAG,IAAI;CAC3B,IAAI;EACF,MAAM,EAAE,WAAW,OAAO,EAAE,QAAQ;EACpC,OAAO;CACT,SAAS,GAAG;EACV,IAAI,sBAAsB,CAAC,GACzB,MAAM,IAAI,mBACR,qEACA,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;AACF;AAiBqD,OAAO,OAC1D,OAAO,YACL,OAAO,QAAQ,GAAG,EAAE,KAAK,CAAC,MAAM,SAAS,CACvC,KACA,KAAK,WAAW,GAAG,IAAI,GAAG,KAAK,UAAU,CAAC,EAAE,MAAM,IACpD,CAAC,CACH,CACF;AAEA,SAAgB,gBAAgB,GAAY,GAAG,OAA8B;CAC3E,OAAO,aAAa,SAAS,iBAAiB,MAAM,SAAS,EAAE,IAAI;AACrE;AAEA,SAAS,iCAAiC,GAAY;CACpD,IAAI,EAAE,aAAa,UAAU,EAAE,UAAU,IACvC,OAAO;CAET,MAAM,EAAC,SAAQ;CAKf,OACE,OAAO,SAAS,YAChB;EACE;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF,EAAE,SAAS,IAAI;AAEnB;AAEA,SAAgB,sBAAsB,GAAY;CAChD,OACE,iCAAiC,CAAC,KAClC,gBACE,GACA,yBACA,gDACA,8BACA,uBACA,wCACA,qBACA,yBACA,2BACA,yBACA,+BACF;AAEJ"}
|
|
1
|
+
{"version":3,"file":"pg.js","names":[],"sources":["../../../../../zero-cache/src/types/pg.ts"],"sourcesContent":["import {Socket} from 'node:net';\nimport {\n PG_CONFIGURATION_LIMIT_EXCEEDED,\n PG_CONNECTION_DOES_NOT_EXIST,\n PG_CONNECTION_EXCEPTION,\n PG_CONNECTION_FAILURE,\n PG_INSUFFICIENT_PRIVILEGE,\n PG_INVALID_AUTHORIZATION_SPECIFICATION,\n PG_INVALID_CATALOG_NAME,\n PG_INVALID_PASSWORD,\n PG_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION,\n PG_TOO_MANY_CONNECTIONS,\n} from '@drdgvhbh/postgres-error-codes';\nimport {PreciseDate} from '@google-cloud/precise-date';\nimport {OID} from '@postgresql-typed/oids';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres, {type Notice, type PostgresType} from 'postgres';\nimport {BigIntJSON, type JSONValue} from '../../../shared/src/bigint-json.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {ConfigurationError} from './configuration-error.ts';\nimport {\n DATE,\n JSON,\n JSONB,\n NUMERIC,\n TIME,\n TIMESTAMP,\n TIMESTAMPTZ,\n TIMETZ,\n} from './pg-types.ts';\n\n// Matches: YEAR-MM-DD HH:MM:SS[.fraction][+-TZ[:MM]][ BC]\nconst pgTimestampRe =\n /^(\\d+)-(\\d{2}-\\d{2}) (\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?)([+-]\\d{1,2}(?::\\d{2})?)?(?: BC)?$/;\n\n// exported for testing.\nexport function timestampToFpMillis(timestamp: string): number {\n if (timestamp === 'infinity') return Infinity;\n if (timestamp === '-infinity') return -Infinity;\n const match = timestamp.match(pgTimestampRe);\n if (!match) {\n throw new Error(`Error parsing ${timestamp}`);\n }\n\n const [, yearStr, monthDay, time, tz] = match;\n const bc = timestamp.endsWith(' BC');\n\n let year = Number(yearStr);\n if (bc) {\n // Postgres: 1 BC = JS year 0, 2 BC = JS year -1, N BC = -(N-1)\n year = -(year - 1);\n }\n\n // Format year as ISO 8601 expanded year if needed.\n // https://tc39.es/ecma262/#sec-expanded-years\n let isoYear: string;\n if (year >= 0 && year <= 9999) {\n isoYear = String(year).padStart(4, '0');\n } else if (year >= 0) {\n isoYear = '+' + String(year).padStart(6, '0');\n } else {\n isoYear = '-' + String(Math.abs(year)).padStart(6, '0');\n }\n\n // Build a UTC ISO string so PreciseDate returns microsecond precision.\n const utcString = `${isoYear}-${monthDay}T${time}Z`;\n\n try {\n const fullTime = new PreciseDate(utcString).getFullTime();\n const millis = Number(fullTime / 1_000_000n);\n const nanos = Number(fullTime % 1_000_000n);\n const ret = millis + nanos * 1e-6; // floating point milliseconds\n\n // Add back in the timezone offset. We passed local time as UTC,\n // so a positive offset means we need to subtract, and vice versa.\n if (tz) {\n const positiveOffset = tz.startsWith('+');\n const [hours, minutes] = tz.split(':');\n const offset =\n Math.abs(Number(hours)) * 60 + (minutes ? Number(minutes) : 0);\n const offsetMillis = offset * 60 * 1_000;\n return positiveOffset ? ret - offsetMillis : ret + offsetMillis;\n }\n return ret;\n } catch (e) {\n throw new Error(`Error parsing ${timestamp}`, {cause: e});\n }\n}\n\nfunction serializeTimestamp(val: unknown): string {\n switch (typeof val) {\n case 'string':\n return val; // Let Postgres parse it\n case 'number': {\n if (Number.isInteger(val)) {\n return new PreciseDate(val).toISOString();\n }\n // Convert floating point to bigint nanoseconds.\n const nanoseconds =\n 1_000_000n * BigInt(Math.trunc(val)) +\n BigInt(Math.trunc((val % 1) * 1e6));\n return new PreciseDate(nanoseconds).toISOString();\n }\n // Note: Don't support bigint inputs until we decide what the semantics are (e.g. micros vs nanos)\n // case 'bigint':\n // return new PreciseDate(val).toISOString();\n default:\n if (val instanceof Date) {\n return val.toISOString();\n }\n }\n throw new Error(`Unsupported type \"${typeof val}\" for timestamp: ${val}`);\n}\n\nconst MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;\n\nfunction serializeTime(x: unknown, type: 'time' | 'timetz'): string {\n switch (typeof x) {\n case 'string':\n return x; // Let Postgres parse it\n case 'number':\n return millisecondsToPostgresTime(x);\n }\n throw new Error(`Unsupported type \"${typeof x}\" for ${type}: ${x}`);\n}\n\nexport function millisecondsToPostgresTime(milliseconds: number): string {\n if (milliseconds < 0) {\n throw new Error('Milliseconds cannot be negative');\n }\n\n if (milliseconds >= MILLISECONDS_PER_DAY) {\n throw new Error(\n `Milliseconds cannot exceed 24 hours (${MILLISECONDS_PER_DAY}ms)`,\n );\n }\n\n milliseconds = Math.floor(milliseconds); // Ensure it's an integer\n\n const totalSeconds = Math.floor(milliseconds / 1000);\n const hours = Math.floor(totalSeconds / 3600);\n const minutes = Math.floor((totalSeconds % 3600) / 60);\n const seconds = totalSeconds % 60;\n const ms = milliseconds % 1000;\n\n return `${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}.${ms.toString().padStart(3, '0')}+00`;\n}\n\n// Regular expression to match HH:MM:SS, HH:MM:SS.mmm, or HH:MM:SS+00 / HH:MM:SS.mmm+00\n// Supports optional timezone offset\nconst timeRegex =\n /^(\\d{1,2}):(\\d{2}):(\\d{2})(?:\\.(\\d{1,6}))?(?:([+-])(\\d{1,2})(?::(\\d{2}))?)?$/;\n\nexport function postgresTimeToMilliseconds(timeString: string): number {\n // Validate basic format\n if (!timeString || typeof timeString !== 'string') {\n throw new Error('Invalid time string: must be a non-empty string');\n }\n\n const match = timeString.match(timeRegex);\n\n if (!match) {\n throw new Error(\n `Invalid time format: \"${timeString}\". Expected HH:MM:SS[.mmm][+|-HH[:MM]]`,\n );\n }\n\n // Extract components\n const hours = parseInt(match[1], 10);\n const minutes = parseInt(match[2], 10);\n const seconds = parseInt(match[3], 10);\n // Handle optional milliseconds, pad right with zeros if needed\n let milliseconds = 0;\n if (match[4]) {\n // Pad microseconds to 6 digits\n const msString = match[4].padEnd(6, '0');\n // slice milliseconds out of the microseconds\n // e.g. 123456 -> 123, 1234 -> 123,\n milliseconds = parseInt(msString.slice(0, 3), 10);\n }\n\n // Validate ranges\n if (hours < 0 || hours > 24) {\n throw new Error(\n `Invalid hours: ${hours}. Must be between 0 and 24 (24 means end of day)`,\n );\n }\n\n if (minutes < 0 || minutes >= 60) {\n throw new Error(`Invalid minutes: ${minutes}. Must be between 0 and 59`);\n }\n\n if (seconds < 0 || seconds >= 60) {\n throw new Error(`Invalid seconds: ${seconds}. Must be between 0 and 59`);\n }\n\n if (milliseconds < 0 || milliseconds >= 1000) {\n throw new Error(\n `Invalid milliseconds: ${milliseconds}. Must be between 0 and 999`,\n );\n }\n\n // Special case: PostgreSQL allows 24:00:00 to represent end of day\n if (hours === 24 && (minutes !== 0 || seconds !== 0 || milliseconds !== 0)) {\n throw new Error(\n 'Invalid time: when hours is 24, minutes, seconds, and milliseconds must be 0',\n );\n }\n\n // Calculate total milliseconds\n let totalMs =\n hours * 3600000 + minutes * 60000 + seconds * 1000 + milliseconds;\n\n // Timezone Offset\n if (match[5]) {\n const sign = match[5] === '+' ? 1 : -1;\n const tzHours = parseInt(match[6], 10);\n const tzMinutes = match[7] ? parseInt(match[7], 10) : 0;\n const offsetMs = sign * (tzHours * 3600000 + tzMinutes * 60000);\n totalMs -= offsetMs;\n }\n\n // Normalize to 0-24h only if outside valid range\n if (totalMs > MILLISECONDS_PER_DAY || totalMs < 0) {\n return (\n ((totalMs % MILLISECONDS_PER_DAY) + MILLISECONDS_PER_DAY) %\n MILLISECONDS_PER_DAY\n );\n }\n\n return totalMs;\n}\n\nfunction dateToUTCMidnight(date: string): number {\n if (date === 'infinity') return Infinity;\n if (date === '-infinity') return -Infinity;\n const d = new Date(date);\n return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate());\n}\n\n/**\n * The (javascript) types of objects that can be returned by our configured\n * Postgres clients. For initial-sync, these comes from the postgres.js client:\n *\n * https://github.com/porsager/postgres/blob/master/src/types.js\n *\n * and for the replication stream these come from the the node-postgres client:\n *\n * https://github.com/brianc/node-pg-types/blob/master/lib/textParsers.js\n */\nexport type PostgresValueType = JSONValue | Uint8Array;\n\nexport type TypeOptions = {\n /**\n * Sends strings directly as JSON values (i.e. without JSON stringification).\n * The application is responsible for ensuring that string inputs for JSON\n * columns are already stringified. Other data types (e.g. objects) will\n * still be stringified by the pg client.\n */\n sendStringAsJson?: boolean;\n};\n\n/**\n * Configures types for the Postgres.js client library (`postgres`).\n *\n * @param jsonAsString Keep JSON / JSONB values as strings instead of parsing.\n */\nexport const postgresTypeConfig = ({sendStringAsJson}: TypeOptions = {}) => ({\n // Type the type IDs as `number` so that Typescript doesn't complain about\n // referencing external types during type inference.\n types: {\n bigint: postgres.BigInt,\n json: {\n to: JSON,\n from: [JSON, JSONB],\n serialize: sendStringAsJson\n ? (x: unknown) => (typeof x === 'string' ? x : BigIntJSON.stringify(x))\n : BigIntJSON.stringify,\n parse: BigIntJSON.parse,\n },\n // Timestamps are converted to PreciseDate objects.\n timestamp: {\n to: TIMESTAMP,\n from: [TIMESTAMP, TIMESTAMPTZ],\n serialize: serializeTimestamp,\n parse: timestampToFpMillis,\n },\n // Times are converted as strings\n time: {\n to: TIME,\n from: [TIME, TIMETZ],\n serialize: (x: unknown) => serializeTime(x, 'time'),\n parse: postgresTimeToMilliseconds,\n },\n\n timetz: {\n to: TIMETZ,\n from: [TIME, TIMETZ],\n serialize: (x: unknown) => serializeTime(x, 'timetz'),\n parse: postgresTimeToMilliseconds,\n },\n\n // The DATE type is stored directly as the PG normalized date string.\n date: {\n to: DATE,\n from: [DATE],\n serialize: (x: string | Date) =>\n (x instanceof Date ? x : new Date(x)).toISOString(),\n parse: dateToUTCMidnight,\n },\n // Returns a `js` number which can lose precision for large numbers.\n // JS number is 53 bits so this should generally not occur.\n // An API will be provided for users to override this type.\n numeric: {\n to: NUMERIC,\n from: [NUMERIC],\n serialize: (x: number) => String(x), // pg expects a string\n parse: (x: string | number) => Number(x),\n },\n },\n});\n\nexport type PostgresDB = postgres.Sql<{\n bigint: bigint;\n json: JSONValue;\n}>;\n\nexport type PostgresTransaction = postgres.TransactionSql<{\n bigint: bigint;\n json: JSONValue;\n}>;\n\n// Guards against half-open TCP connections, which otherwise hang the pool\n// indefinitely: a proxy can keep the client-facing socket alive (ACKing TCP\n// keepalives) after its backend is gone, so no 'close' or 'error' event ever\n// fires and statements like BEGIN / COMMIT await forever.\n// See https://github.com/porsager/postgres/issues/1089.\n//\n// If no bytes are read or written for this long, the socket is reset, which\n// rejects all in-flight queries and lets the pool recover. Wire activity\n// resets the timer, so streaming operations (e.g. COPY) are safe, but\n// statements that legitimately compute silently for longer (e.g. index\n// builds in migrations) must raise this. Settable as an emergency measure\n// (0 disables); deliberately not exposed as a server option.\nconst SOCKET_INACTIVITY_TIMEOUT_MS = parseInt(\n process.env.ZERO_PG_SOCKET_INACTIVITY_TIMEOUT ?? '120000',\n);\n\ntype SocketFactoryOptions = {\n host: string[];\n port: number[];\n path?: string | false;\n};\n\n/**\n * Creates the custom socket factory passed to postgres.js via its (untyped)\n * `socket` option. postgres.js does not call `connect()` on custom sockets,\n * and its TLS upgrade reads `socket.host` for the SNI servername, so the\n * factory must handle both itself.\n *\n * Note: unlike postgres.js's internal socket, reconnects do not rotate\n * through multiple hosts; multi-host URIs are not used by the zero-cache.\n *\n * Exported for testing.\n */\nexport function inactivityTimeoutSocket(\n lc: LogContext,\n timeoutMs = SOCKET_INACTIVITY_TIMEOUT_MS,\n) {\n return (options: SocketFactoryOptions): Socket => {\n const socket = new Socket();\n const target = options.path\n ? options.path\n : `${options.host[0]}:${options.port[0]}`;\n if (timeoutMs > 0) {\n // socket.setTimeout() cannot be used here: it registers its callback\n // as a 'timeout' event listener, and postgres.js removes all listeners\n // from the raw socket when upgrading it to TLS (removeAllListeners()\n // in secure()), which would silently disarm the watchdog on every TLS\n // connection. An interval is not a socket listener, so it survives the\n // upgrade, and encrypted traffic still flows through this raw socket,\n // advancing its byte counters.\n //\n // The counters are sampled once per timeout period, so a half-open\n // socket is reset after one to two timeout periods of inactivity.\n let lastActivity = -1;\n const watchdog = setInterval(() => {\n if (socket.destroyed) {\n clearInterval(watchdog);\n return;\n }\n const activity = socket.bytesRead + socket.bytesWritten;\n if (activity === lastActivity) {\n lc.warn?.(\n `resetting connection to ${target} after ${timeoutMs} ms of inactivity ` +\n `(likely half-open); in-flight queries will be rejected`,\n );\n clearInterval(watchdog);\n socket.resetAndDestroy();\n } else {\n lastActivity = activity;\n }\n }, timeoutMs);\n watchdog.unref();\n // Best-effort cleanup; removed on TLS upgrade, in which case the\n // destroyed check above clears the (unref'ed) interval instead.\n socket.once('close', () => clearInterval(watchdog));\n }\n if (options.path) {\n socket.connect(options.path);\n } else {\n const [host] = options.host;\n const [port] = options.port;\n socket.connect(port, host);\n // Read by postgres.js for the TLS SNI servername.\n Object.assign(socket, {host, port});\n }\n return socket;\n };\n}\n\nexport function pgClient(\n lc: LogContext,\n connectionURI: string,\n // A descriptive name for the code that is making the connection, so that we\n // can debug things like deadlocks.\n applicationName: string,\n options?: postgres.Options<{\n bigint: PostgresType<bigint>;\n json: PostgresType<JSONValue>;\n }>,\n opts?: TypeOptions,\n): PostgresDB {\n applicationName = `zero-${applicationName}`;\n\n // Postgres carries row values in the `detail`, `hint`, `where` and\n // `internal_query` fields -- e.g. a unique violation reports\n // `Key (email)=(someone@example.com) already exists` in `detail`. Forward\n // only the severity, SQLSTATE and schema location; never the Notice itself.\n const safeNotice = (n: Notice) => ({\n severity: n.severity,\n code: n.code,\n message: n.message,\n schema: n.schema_name,\n table: n.table_name,\n column: n.column_name,\n constraint: n.constraint_name,\n routine: n.routine,\n });\n\n const onnotice = (n: Notice) => {\n // https://www.postgresql.org/docs/current/plpgsql-errors-and-messages.html#PLPGSQL-STATEMENTS-RAISE\n switch (n.severity) {\n case 'NOTICE':\n return; // silenced\n case 'DEBUG':\n lc.debug?.('pg notice', safeNotice(n));\n return;\n case 'WARNING':\n lc.warn?.('pg notice', safeNotice(n));\n return;\n case 'EXCEPTION':\n lc.error?.('pg notice', safeNotice(n));\n return;\n case 'LOG':\n case 'INFO':\n default:\n lc.info?.('pg notice', safeNotice(n));\n }\n };\n const url = new URL(connectionURI);\n const sslFlag =\n url.searchParams.get('ssl') ?? url.searchParams.get('sslmode') ?? 'prefer';\n\n let ssl: boolean | 'prefer' | {rejectUnauthorized: boolean};\n if (sslFlag === 'disable' || sslFlag === 'false') {\n ssl = false;\n } else if (sslFlag === 'no-verify') {\n ssl = {rejectUnauthorized: false};\n } else {\n ssl = sslFlag as 'prefer';\n }\n\n // Set connections to expire between 5 and 10 minutes to free up state on PG.\n const maxLifetimeSeconds = randInt(5 * 60, 10 * 60);\n const providedConnection = options?.connection ?? {};\n\n // postgres.js supports a custom socket factory via the `socket` option,\n // but it is missing from its type declarations, hence the untyped spread.\n const socketFactory = {\n socket: inactivityTimeoutSocket(lc.withContext('appName', applicationName)),\n };\n\n return postgres(connectionURI, {\n ...postgresTypeConfig(opts),\n onnotice,\n ['max_lifetime']: maxLifetimeSeconds,\n // Close idle connections cleanly before the socket inactivity timer\n // (which cannot distinguish an idle pool connection from a hung query)\n // resets them.\n ['idle_timeout']: 60,\n ssl,\n ...socketFactory,\n ...options,\n connection: {\n ...providedConnection,\n ['application_name']: applicationName,\n },\n });\n}\n\nexport async function connectPgClient(\n ...args: Parameters<typeof pgClient>\n): Promise<PostgresDB> {\n const db = pgClient(...args);\n try {\n await db`SELECT 1`.simple().execute();\n return db;\n } catch (e) {\n if (isPostgresConfigError(e)) {\n throw new ConfigurationError(\n 'Unable to connect to Postgres. Check your database configuration.',\n {cause: e},\n );\n }\n throw e;\n }\n}\n\n/**\n * Disables any statement_timeout for the current transaction. By default,\n * Postgres does not impose a statement timeout, but some users and providers\n * set one at the database level (even though it is explicitly discouraged by\n * the Postgres documentation).\n *\n * Zero logic in particular often does not fit into the category of general\n * application logic; for potentially long-running operations like migrations\n * and background cleanup, the statement timeout should be disabled to prevent\n * these operations from timing out.\n */\nexport function disableStatementTimeout(sql: PostgresTransaction) {\n void sql`SET LOCAL statement_timeout = 0;`.execute().catch(() => {});\n}\n\nexport const typeNameByOID: Record<number, string> = Object.freeze(\n Object.fromEntries(\n Object.entries(OID).map(([name, oid]) => [\n oid,\n name.startsWith('_') ? `${name.substring(1)}[]` : name,\n ]),\n ),\n);\n\nexport function isPostgresError(e: unknown, ...codes: [string, ...string[]]) {\n return e instanceof postgres.PostgresError && codes.includes(e.code);\n}\n\nfunction isPostgresStartupConnectionError(e: unknown) {\n if (!(e instanceof Error) || !('code' in e)) {\n return false;\n }\n const {code} = e;\n // postgres.js throws plain Error instances for connection failures that\n // happen before Postgres can return a SQLSTATE. These are still startup\n // configuration problems for Zero: wrong host/port, unreachable DB, DNS,\n // TLS certificate mismatch, etc.\n return (\n typeof code === 'string' &&\n [\n 'CONNECT_TIMEOUT',\n 'ECONNREFUSED',\n 'ECONNRESET',\n 'EHOSTUNREACH',\n 'ENETUNREACH',\n 'ENOTFOUND',\n 'EAI_AGAIN',\n 'ETIMEDOUT',\n 'ERR_TLS_CERT_ALTNAME_INVALID',\n 'SELF_SIGNED_CERT_IN_CHAIN',\n 'DEPTH_ZERO_SELF_SIGNED_CERT',\n 'UNABLE_TO_VERIFY_LEAF_SIGNATURE',\n ].includes(code)\n );\n}\n\nexport function isPostgresConfigError(e: unknown) {\n return (\n isPostgresStartupConnectionError(e) ||\n isPostgresError(\n e,\n PG_CONNECTION_EXCEPTION,\n PG_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION,\n PG_CONNECTION_DOES_NOT_EXIST,\n PG_CONNECTION_FAILURE,\n PG_INVALID_AUTHORIZATION_SPECIFICATION,\n PG_INVALID_PASSWORD,\n PG_INVALID_CATALOG_NAME,\n PG_INSUFFICIENT_PRIVILEGE,\n PG_TOO_MANY_CONNECTIONS,\n PG_CONFIGURATION_LIMIT_EXCEEDED,\n )\n );\n}\n"],"mappings":";;;;;;;;;;AAgCA,IAAM,gBACJ;AAGF,SAAgB,oBAAoB,WAA2B;CAC7D,IAAI,cAAc,YAAY,OAAO;CACrC,IAAI,cAAc,aAAa,OAAO;CACtC,MAAM,QAAQ,UAAU,MAAM,aAAa;CAC3C,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,WAAW;CAG9C,MAAM,GAAG,SAAS,UAAU,MAAM,MAAM;CACxC,MAAM,KAAK,UAAU,SAAS,KAAK;CAEnC,IAAI,OAAO,OAAO,OAAO;CACzB,IAAI,IAEF,OAAO,EAAE,OAAO;CAKlB,IAAI;CACJ,IAAI,QAAQ,KAAK,QAAQ,MACvB,UAAU,OAAO,IAAI,EAAE,SAAS,GAAG,GAAG;MACjC,IAAI,QAAQ,GACjB,UAAU,MAAM,OAAO,IAAI,EAAE,SAAS,GAAG,GAAG;MAE5C,UAAU,MAAM,OAAO,KAAK,IAAI,IAAI,CAAC,EAAE,SAAS,GAAG,GAAG;CAIxD,MAAM,YAAY,GAAG,QAAQ,GAAG,SAAS,GAAG,KAAK;CAEjD,IAAI;EACF,MAAM,WAAW,IAAI,YAAY,SAAS,EAAE,YAAY;EAGxD,MAAM,MAFS,OAAO,WAAW,QAErB,IADE,OAAO,WAAW,QACX,IAAQ;EAI7B,IAAI,IAAI;GACN,MAAM,iBAAiB,GAAG,WAAW,GAAG;GACxC,MAAM,CAAC,OAAO,WAAW,GAAG,MAAM,GAAG;GAGrC,MAAM,gBADJ,KAAK,IAAI,OAAO,KAAK,CAAC,IAAI,MAAM,UAAU,OAAO,OAAO,IAAI,MAChC,KAAK;GACnC,OAAO,iBAAiB,MAAM,eAAe,MAAM;EACrD;EACA,OAAO;CACT,SAAS,GAAG;EACV,MAAM,IAAI,MAAM,iBAAiB,aAAa,EAAC,OAAO,EAAC,CAAC;CAC1D;AACF;AAEA,SAAS,mBAAmB,KAAsB;CAChD,QAAQ,OAAO,KAAf;EACE,KAAK,UACH,OAAO;EACT,KAAK;GACH,IAAI,OAAO,UAAU,GAAG,GACtB,OAAO,IAAI,YAAY,GAAG,EAAE,YAAY;GAM1C,OAAO,IAAI,YAFT,WAAa,OAAO,KAAK,MAAM,GAAG,CAAC,IACnC,OAAO,KAAK,MAAO,MAAM,IAAK,GAAG,CAAC,CACF,EAAE,YAAY;EAKlD,SACE,IAAI,eAAe,MACjB,OAAO,IAAI,YAAY;CAE7B;CACA,MAAM,IAAI,MAAM,qBAAqB,OAAO,IAAI,mBAAmB,KAAK;AAC1E;AAEA,IAAM,uBAAuB,OAAU,KAAK;AAE5C,SAAS,cAAc,GAAY,MAAiC;CAClE,QAAQ,OAAO,GAAf;EACE,KAAK,UACH,OAAO;EACT,KAAK,UACH,OAAO,2BAA2B,CAAC;CACvC;CACA,MAAM,IAAI,MAAM,qBAAqB,OAAO,EAAE,QAAQ,KAAK,IAAI,GAAG;AACpE;AAEA,SAAgB,2BAA2B,cAA8B;CACvE,IAAI,eAAe,GACjB,MAAM,IAAI,MAAM,iCAAiC;CAGnD,IAAI,gBAAgB,sBAClB,MAAM,IAAI,MACR,wCAAwC,qBAAqB,IAC/D;CAGF,eAAe,KAAK,MAAM,YAAY;CAEtC,MAAM,eAAe,KAAK,MAAM,eAAe,GAAI;CACnD,MAAM,QAAQ,KAAK,MAAM,eAAe,IAAI;CAC5C,MAAM,UAAU,KAAK,MAAO,eAAe,OAAQ,EAAE;CACrD,MAAM,UAAU,eAAe;CAC/B,MAAM,KAAK,eAAe;CAE1B,OAAO,GAAG,MAAM,SAAS,EAAE,SAAS,GAAG,GAAG,EAAE,GAAG,QAAQ,SAAS,EAAE,SAAS,GAAG,GAAG,EAAE,GAAG,QAAQ,SAAS,EAAE,SAAS,GAAG,GAAG,EAAE,GAAG,GAAG,SAAS,EAAE,SAAS,GAAG,GAAG,EAAE;AAC9J;AAIA,IAAM,YACJ;AAEF,SAAgB,2BAA2B,YAA4B;CAErE,IAAI,CAAC,cAAc,OAAO,eAAe,UACvC,MAAM,IAAI,MAAM,iDAAiD;CAGnE,MAAM,QAAQ,WAAW,MAAM,SAAS;CAExC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,yBAAyB,WAAW,uCACtC;CAIF,MAAM,QAAQ,SAAS,MAAM,IAAI,EAAE;CACnC,MAAM,UAAU,SAAS,MAAM,IAAI,EAAE;CACrC,MAAM,UAAU,SAAS,MAAM,IAAI,EAAE;CAErC,IAAI,eAAe;CACnB,IAAI,MAAM,IAAI;EAEZ,MAAM,WAAW,MAAM,GAAG,OAAO,GAAG,GAAG;EAGvC,eAAe,SAAS,SAAS,MAAM,GAAG,CAAC,GAAG,EAAE;CAClD;CAGA,IAAI,QAAQ,KAAK,QAAQ,IACvB,MAAM,IAAI,MACR,kBAAkB,MAAM,iDAC1B;CAGF,IAAI,UAAU,KAAK,WAAW,IAC5B,MAAM,IAAI,MAAM,oBAAoB,QAAQ,2BAA2B;CAGzE,IAAI,UAAU,KAAK,WAAW,IAC5B,MAAM,IAAI,MAAM,oBAAoB,QAAQ,2BAA2B;CAGzE,IAAI,eAAe,KAAK,gBAAgB,KACtC,MAAM,IAAI,MACR,yBAAyB,aAAa,4BACxC;CAIF,IAAI,UAAU,OAAO,YAAY,KAAK,YAAY,KAAK,iBAAiB,IACtE,MAAM,IAAI,MACR,8EACF;CAIF,IAAI,UACF,QAAQ,OAAU,UAAU,MAAQ,UAAU,MAAO;CAGvD,IAAI,MAAM,IAAI;EACZ,MAAM,OAAO,MAAM,OAAO,MAAM,IAAI;EACpC,MAAM,UAAU,SAAS,MAAM,IAAI,EAAE;EACrC,MAAM,YAAY,MAAM,KAAK,SAAS,MAAM,IAAI,EAAE,IAAI;EACtD,MAAM,WAAW,QAAQ,UAAU,OAAU,YAAY;EACzD,WAAW;CACb;CAGA,IAAI,UAAU,wBAAwB,UAAU,GAC9C,QACI,UAAU,uBAAwB,wBACpC;CAIJ,OAAO;AACT;AAEA,SAAS,kBAAkB,MAAsB;CAC/C,IAAI,SAAS,YAAY,OAAO;CAChC,IAAI,SAAS,aAAa,OAAO;CACjC,MAAM,IAAI,IAAI,KAAK,IAAI;CACvB,OAAO,KAAK,IAAI,EAAE,eAAe,GAAG,EAAE,YAAY,GAAG,EAAE,WAAW,CAAC;AACrE;;;;;;AA6BA,IAAa,sBAAsB,EAAC,qBAAiC,CAAC,OAAO,EAG3E,OAAO;CACL,QAAQ,SAAS;CACjB,MAAM;EACJ,IAAA;EACA,MAAM,CAAA,KAAO,KAAK;EAClB,WAAW,oBACN,MAAgB,OAAO,MAAM,WAAW,IAAI,WAAW,UAAU,CAAC,IACnE,WAAW;EACf,OAAO,WAAW;CACpB;CAEA,WAAW;EACT,IAAI;EACJ,MAAM,CAAC,WAAW,WAAW;EAC7B,WAAW;EACX,OAAO;CACT;CAEA,MAAM;EACJ,IAAI;EACJ,MAAM,CAAC,MAAM,MAAM;EACnB,YAAY,MAAe,cAAc,GAAG,MAAM;EAClD,OAAO;CACT;CAEA,QAAQ;EACN,IAAI;EACJ,MAAM,CAAC,MAAM,MAAM;EACnB,YAAY,MAAe,cAAc,GAAG,QAAQ;EACpD,OAAO;CACT;CAGA,MAAM;EACJ,IAAI;EACJ,MAAM,CAAC,IAAI;EACX,YAAY,OACT,aAAa,OAAO,IAAI,IAAI,KAAK,CAAC,GAAG,YAAY;EACpD,OAAO;CACT;CAIA,SAAS;EACP,IAAI;EACJ,MAAM,CAAC,OAAO;EACd,YAAY,MAAc,OAAO,CAAC;EAClC,QAAQ,MAAuB,OAAO,CAAC;CACzC;AACF,EACF;AAwBA,IAAM,+BAA+B,SACnC,QAAQ,IAAI,qCAAqC,QACnD;;;;;;;;;;;;AAmBA,SAAgB,wBACd,IACA,YAAY,8BACZ;CACA,QAAQ,YAA0C;EAChD,MAAM,SAAS,IAAI,OAAO;EAC1B,MAAM,SAAS,QAAQ,OACnB,QAAQ,OACR,GAAG,QAAQ,KAAK,GAAG,GAAG,QAAQ,KAAK;EACvC,IAAI,YAAY,GAAG;GAWjB,IAAI,eAAe;GACnB,MAAM,WAAW,kBAAkB;IACjC,IAAI,OAAO,WAAW;KACpB,cAAc,QAAQ;KACtB;IACF;IACA,MAAM,WAAW,OAAO,YAAY,OAAO;IAC3C,IAAI,aAAa,cAAc;KAC7B,GAAG,OACD,2BAA2B,OAAO,SAAS,UAAU,yEAEvD;KACA,cAAc,QAAQ;KACtB,OAAO,gBAAgB;IACzB,OACE,eAAe;GAEnB,GAAG,SAAS;GACZ,SAAS,MAAM;GAGf,OAAO,KAAK,eAAe,cAAc,QAAQ,CAAC;EACpD;EACA,IAAI,QAAQ,MACV,OAAO,QAAQ,QAAQ,IAAI;OACtB;GACL,MAAM,CAAC,QAAQ,QAAQ;GACvB,MAAM,CAAC,QAAQ,QAAQ;GACvB,OAAO,QAAQ,MAAM,IAAI;GAEzB,OAAO,OAAO,QAAQ;IAAC;IAAM;GAAI,CAAC;EACpC;EACA,OAAO;CACT;AACF;AAEA,SAAgB,SACd,IACA,eAGA,iBACA,SAIA,MACY;CACZ,kBAAkB,QAAQ;CAM1B,MAAM,cAAc,OAAe;EACjC,UAAU,EAAE;EACZ,MAAM,EAAE;EACR,SAAS,EAAE;EACX,QAAQ,EAAE;EACV,OAAO,EAAE;EACT,QAAQ,EAAE;EACV,YAAY,EAAE;EACd,SAAS,EAAE;CACb;CAEA,MAAM,YAAY,MAAc;EAE9B,QAAQ,EAAE,UAAV;GACE,KAAK,UACH;GACF,KAAK;IACH,GAAG,QAAQ,aAAa,WAAW,CAAC,CAAC;IACrC;GACF,KAAK;IACH,GAAG,OAAO,aAAa,WAAW,CAAC,CAAC;IACpC;GACF,KAAK;IACH,GAAG,QAAQ,aAAa,WAAW,CAAC,CAAC;IACrC;GAGF,SACE,GAAG,OAAO,aAAa,WAAW,CAAC,CAAC;EACxC;CACF;CACA,MAAM,MAAM,IAAI,IAAI,aAAa;CACjC,MAAM,UACJ,IAAI,aAAa,IAAI,KAAK,KAAK,IAAI,aAAa,IAAI,SAAS,KAAK;CAEpE,IAAI;CACJ,IAAI,YAAY,aAAa,YAAY,SACvC,MAAM;MACD,IAAI,YAAY,aACrB,MAAM,EAAC,oBAAoB,MAAK;MAEhC,MAAM;CAIR,MAAM,qBAAqB,QAAQ,KAAQ,GAAO;CAClD,MAAM,qBAAqB,SAAS,cAAc,CAAC;CAInD,MAAM,gBAAgB,EACpB,QAAQ,wBAAwB,GAAG,YAAY,WAAW,eAAe,CAAC,EAC5E;CAEA,OAAO,SAAS,eAAe;EAC7B,GAAG,mBAAmB,IAAI;EAC1B;GACC,iBAAiB;GAIjB,iBAAiB;EAClB;EACA,GAAG;EACH,GAAG;EACH,YAAY;GACV,GAAG;IACF,qBAAqB;EACxB;CACF,CAAC;AACH;AAEA,eAAsB,gBACpB,GAAG,MACkB;CACrB,MAAM,KAAK,SAAS,GAAG,IAAI;CAC3B,IAAI;EACF,MAAM,EAAE,WAAW,OAAO,EAAE,QAAQ;EACpC,OAAO;CACT,SAAS,GAAG;EACV,IAAI,sBAAsB,CAAC,GACzB,MAAM,IAAI,mBACR,qEACA,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;AACF;AAiBqD,OAAO,OAC1D,OAAO,YACL,OAAO,QAAQ,GAAG,EAAE,KAAK,CAAC,MAAM,SAAS,CACvC,KACA,KAAK,WAAW,GAAG,IAAI,GAAG,KAAK,UAAU,CAAC,EAAE,MAAM,IACpD,CAAC,CACH,CACF;AAEA,SAAgB,gBAAgB,GAAY,GAAG,OAA8B;CAC3E,OAAO,aAAa,SAAS,iBAAiB,MAAM,SAAS,EAAE,IAAI;AACrE;AAEA,SAAS,iCAAiC,GAAY;CACpD,IAAI,EAAE,aAAa,UAAU,EAAE,UAAU,IACvC,OAAO;CAET,MAAM,EAAC,SAAQ;CAKf,OACE,OAAO,SAAS,YAChB;EACE;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF,EAAE,SAAS,IAAI;AAEnB;AAEA,SAAgB,sBAAsB,GAAY;CAChD,OACE,iCAAiC,CAAC,KAClC,gBACE,GACA,yBACA,gDACA,8BACA,uBACA,wCACA,qBACA,yBACA,2BACA,yBACA,+BACF;AAEJ"}
|
|
@@ -14,16 +14,11 @@ export type Source<T> = AsyncIterable<T> & {
|
|
|
14
14
|
*/
|
|
15
15
|
cancel: (err?: Error) => void;
|
|
16
16
|
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* This is similar to using `Promise.race([other, sourceDone])`, but
|
|
22
|
-
* in a memory safe way, as repeatedly calling `Promise.race([ ... ])` with
|
|
23
|
-
* a long-lived Promise results in leaking memory via a growing list of
|
|
24
|
-
* `then()` callbacks (https://github.com/nodejs/node/issues/17469).
|
|
17
|
+
* An AbortSignal that can be used to listen for termination or
|
|
18
|
+
* race a short-lived promise against the termination of the iterable
|
|
19
|
+
* (via `promiseOrAbort()`)
|
|
25
20
|
*/
|
|
26
|
-
|
|
21
|
+
readonly signal: AbortSignal;
|
|
27
22
|
/**
|
|
28
23
|
* The presence of a `pipeline` iterable allows the usual "consumed-on-iterate" semantics
|
|
29
24
|
* to be overridden.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/streams.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,QAAQ,EAGR,KAAK,aAAa,EACnB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAKL,KAAK,SAAS,EACf,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAa,KAAK,SAAS,EAAC,MAAM,oCAAoC,CAAC;AAE9E,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,EAAC,YAAY,EAAE,KAAK,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAW7D,MAAM,MAAM,MAAM,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG;IACzC;;;;;;OAMG;IACH,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;IAE9B
|
|
1
|
+
{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/streams.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,QAAQ,EAGR,KAAK,aAAa,EACnB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAKL,KAAK,SAAS,EACf,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAa,KAAK,SAAS,EAAC,MAAM,oCAAoC,CAAC;AAE9E,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,EAAC,YAAY,EAAE,KAAK,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAW7D,MAAM,MAAM,MAAM,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG;IACzC;;;;;;OAMG;IACH,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;IAE9B;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAE7B;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,IAAI,CAAA;KAAC,CAAC,GAAG,SAAS,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,IAAI,CAAC,CAAC,IAAI;IACpB,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC;CACxB,CAAC;AAEF;;;GAGG;AAIH,wBAAgB,MAAM,CAAC,EAAE,SAAS,SAAS,EAAE,GAAG,SAAS,SAAS,EAChE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EACpB,UAAU,GAAE,OAAO,CAAC,GAAG,CAAM,EAC7B,SAAS,GAAE,OAAO,CAAC,EAAE,CAAM,EAC3B,aAAa,GAAE,aAAkB,GAChC;IAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAA;CAAC,CAwE9C;AAED,KAAK,WAAW,CAAC,CAAC,IAAI;IACpB,MAAM,EAAE,QAAQ,CAAC;IACjB,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACtB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,wBAAgB,IAAI,CAAC,CAAC,EAAE,EAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,cAAc,EAAC,EAAE,WAAW,CAAC,CAAC,CAAC,QAsD5E;AAUD,gFAAgF;AAChF,MAAM,MAAM,UAAU,CAAC,CAAC,IAAI;IAC1B,IAAI,EAAE,CAAC,CAAC;IACR,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,wBAAgB,SAAS,CAAC,CAAC,SAAS,SAAS,EAC3C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EACjB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CAEf;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EACtB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CAEf;AAmED,wBAAgB,QAAQ,CAAC,CAAC,SAAS,SAAS,EAC1C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAChB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAEpB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,SAAS,SAAS,EACrD,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAChB,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAKhC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streams.js","names":["#lc","#ws","#closeStream","#messageHandler","#connected","#handleOpen","#handleClose","#handleError","#conn"],"sources":["../../../../../zero-cache/src/types/streams.ts"],"sourcesContent":["import {\n pipeline,\n Readable,\n Transform,\n Writable,\n type DuplexOptions,\n} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n createWebSocketStream,\n type CloseEvent,\n type ErrorEvent,\n type MessageEvent,\n type WebSocket,\n} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {BigIntJSON, type JSONValue} from '../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Subscription, type Options} from './subscription.ts';\nimport {\n closeWithError,\n expectPingsForLiveness,\n sendPingsForLiveness,\n} from './ws.ts';\n\n// Consistent with Postgres keepalives, and shorter than the\n// commonly used default idle timeout of 1 minute.\nconst PING_INTERVAL_MS = 30_000;\n\nexport type Source<T> = AsyncIterable<T> & {\n /**\n * Immediately terminates all current iterations (i.e. {@link AsyncIterator.next next()})\n * will return `{value: undefined, done: true}`), and prevents any subsequent iterations\n * from yielding any values.\n *\n * @param err Terminate the iteration by throwing the `err` instead.\n */\n cancel: (err?: Error) => void;\n\n /**\n * Resolves the result of the specified `other` Promise, or a\n * resolved/rejected Promise when the Source is terminated or\n * rejects.\n *\n * This is similar to using `Promise.race([other, sourceDone])`, but\n * in a memory safe way, as repeatedly calling `Promise.race([ ... ])` with\n * a long-lived Promise results in leaking memory via a growing list of\n * `then()` callbacks (https://github.com/nodejs/node/issues/17469).\n */\n doneOr<R>(other: Promise<R>): Promise<void | R>;\n\n /**\n * The presence of a `pipeline` iterable allows the usual \"consumed-on-iterate\" semantics\n * to be overridden.\n *\n * This is suitable for transport layers that serialize messages across processes, such\n * as the {@link streamOut()} method; pipelining allows the transport to send messages\n * as they arrive without waiting for the previous message to be acked, streaming\n * them to the receiving process where they are presumably queued and processed without\n * a per-message ack delay. The receiving end of the transport then responds with acks\n * asynchronously as the receiving end processes the messages.\n */\n pipeline?: AsyncIterable<{value: T; consumed: () => void}> | undefined;\n};\n\nexport type Sink<T> = {\n push(message: T): void;\n};\n\n/**\n * Back-pressure-aware transformation of a WebSocket into\n * upstream and downstream {@link Subscription} objects.\n */\n// TODO: Change {@link streamIn} and {@link streamOut} to use this\n// under the covers so that internal communication is also\n// responsive to backpressure.\nexport function stream<In extends JSONValue, Out extends JSONValue>(\n lc: LogContext,\n ws: WebSocket,\n inSchema: v.Type<In>,\n outOptions: Options<Out> = {},\n inOptions: Options<In> = {},\n streamOptions: DuplexOptions = {},\n): {outstream: Sink<Out>; instream: Source<In>} {\n const endpoint = ws.url ?? 'client';\n function close(err?: unknown) {\n if (ws.readyState !== ws.CLOSED && ws.readyState !== ws.CLOSING) {\n if (err) {\n closeWithError(lc, ws, err);\n } else {\n lc.info?.(`closing connection to ${endpoint}`);\n ws.close();\n }\n }\n }\n\n const instream = Subscription.create<In>({\n ...inOptions,\n cleanup: (unconsumed, err) => {\n inOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n const outstream = Subscription.create<Out>({\n ...outOptions,\n cleanup: (unconsumed, err) => {\n outOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n\n const duplex = createWebSocketStream(ws, {\n ...streamOptions,\n decodeStrings: false,\n });\n\n // Outgoing transform.\n function streamOut() {\n // Mainly used for verifying that back-pressure kicks in tests.\n duplex.on('drain', () => lc.debug?.(`drained messages to ${endpoint}`));\n\n pipeline(\n Readable.from(outstream),\n new Transform({\n objectMode: true,\n transform: (msg, _encoding, callback) =>\n callback(null, BigIntJSON.stringify(msg)),\n }),\n duplex,\n err => (err ? outstream.fail(err) : outstream.cancel()),\n );\n }\n\n if (ws.readyState === ws.CONNECTING) {\n ws.on('open', () => {\n lc.info?.(`connected to ${endpoint}`);\n streamOut();\n });\n } else {\n streamOut();\n }\n\n // Incoming transform.\n pipe({\n source: duplex,\n sink: instream,\n parse: chunk => {\n const json = BigIntJSON.parse(chunk.toString());\n return v.parse(json, inSchema, 'passthrough');\n },\n });\n\n sendPingsForLiveness(lc, ws, PING_INTERVAL_MS);\n\n return {outstream, instream};\n}\n\ntype PipeOptions<T> = {\n source: Readable;\n sink: Subscription<T>;\n parse: (buffer: Buffer) => T | null;\n bufferMessages?: number;\n};\n\nexport function pipe<T>({source, sink, parse, bufferMessages}: PipeOptions<T>) {\n bufferMessages ??= 0;\n assert(bufferMessages >= 0, 'bufferMessages must be non-negative');\n const pending: Promise<unknown>[] = [];\n\n pipeline(\n source,\n new Writable({\n decodeStrings: false,\n write: (chunk, _encoding, callback) => {\n let msg: T | null;\n try {\n if ((msg = parse(chunk)) === null) {\n callback();\n return;\n }\n } catch (err) {\n callback(ensureError(err));\n return;\n }\n // Inbound backpressure is exerted by unconsumed messages in the\n // subscription. A buffer can be used to allow messages to queue up in\n // in the Subscription object, which allows the consumer to \"peek\" at\n // whether there are more messages immediately available\n // (via {@link Subscription.queued}.\n const {result} = sink.push(msg);\n pending.push(result);\n void result.then(() => pending.shift());\n\n if (pending.length <= bufferMessages) {\n // immediately allow more messages\n callback();\n } else {\n // wait for the oldest result in the pending queue\n pending[0].then(\n () => callback(),\n err => callback(ensureError(err)),\n );\n }\n },\n destroy: (err, callback) => {\n if (err) {\n sink.fail(ensureError(err));\n }\n // Otherwise, final will handle the cancel.\n callback();\n },\n final: callback => {\n sink.cancel();\n callback();\n },\n }),\n err => (err ? sink.fail(err) : sink.cancel()),\n );\n}\n\nfunction ensureError(err: unknown) {\n return err instanceof Error ? err : new Error(String(err));\n}\n\nconst ackSchema = v.object({ack: v.number()});\n\ntype Ack = v.Infer<typeof ackSchema>;\n\n/** A parsed JSON value paired with the exact source text it was parsed from. */\nexport type ParsedJSON<T> = {\n data: T;\n json: string;\n};\n\nexport function streamOut<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, BigIntJSON.stringify);\n}\n\n/**\n * Streams out a `Source` for which messages are already stringified JSON.\n */\nexport function streamOutStringified(\n lc: LogContext,\n source: Source<string>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, json => json);\n}\n\nasync function streamOutInternal<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n stringify: (payload: T) => string,\n): Promise<void> {\n sendPingsForLiveness(lc, sink, PING_INTERVAL_MS);\n\n const closer = WebSocketCloser.forSource(lc, sink, source);\n\n const acks = new Queue<Ack>();\n sink.addEventListener('message', ({data}) => {\n try {\n if (typeof data !== 'string') {\n throw new Error('Expected string message');\n }\n acks.enqueue(v.parse(JSON.parse(data), ackSchema));\n } catch (e) {\n lc.error?.(`error parsing ack`, e);\n closer.close(e);\n }\n });\n\n try {\n let nextID = 0;\n const {pipeline} = source;\n if (pipeline) {\n lc.debug?.(`started pipelined outbound stream`);\n for await (const {value: msg, consumed} of pipeline) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`pipelining`, data);\n sink.send(data);\n\n void (async () => {\n const {ack} = await acks.dequeue();\n // lc.debug?.(`received ack`, ack);\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n consumed();\n })();\n }\n } else {\n lc.debug?.(`started synchronous outbound stream`);\n for await (const msg of source) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`sending`, data);\n sink.send(data);\n\n const {ack} = await acks.dequeue();\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n }\n }\n closer.close();\n } catch (e) {\n closer.close(e);\n }\n}\n\nexport function streamIn<T extends JSONValue>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n): Promise<Source<T>> {\n return streamInInternal(lc, source, schema, data => data);\n}\n\n/**\n * Streams in messages sent by {@link streamOutStringified}, preserving the\n * exact application-level JSON alongside its parsed and validated value.\n */\nexport function streamInStringified<T extends JSONValue>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n): Promise<Source<ParsedJSON<T>>> {\n return streamInInternal(lc, source, schema, (data, frame, id) => ({\n data,\n json: extractStringifiedMessage(frame, id),\n }));\n}\n\nasync function streamInInternal<T extends JSONValue, Out>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n transform: (data: T, frame: string, id: number) => Out,\n): Promise<Source<Out>> {\n expectPingsForLiveness(lc, source, PING_INTERVAL_MS);\n\n const streamedSchema = v.object({\n msg: schema,\n id: v.number(),\n });\n\n const sink: Subscription<Out, {id: number; data: Out}> = new Subscription<\n Out,\n {id: number; data: Out}\n >(\n {\n consumed: ({id}) => source.send(JSON.stringify({ack: id} satisfies Ack)),\n cleanup: () => closer.close(),\n },\n ({data}) => data,\n );\n\n const closer = WebSocketCloser.forSink(lc, source, sink, handleMessage);\n\n function handleMessage(event: MessageEvent) {\n const data = event.data.toString();\n if (!sink.active) {\n lc.warn?.('dropping ws message received after close', data);\n return;\n }\n try {\n const value = BigIntJSON.parse(data);\n const msg = v.parse(value, streamedSchema, 'passthrough');\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`received`, data);\n sink.push({id: msg.id, data: transform(msg.msg, data, msg.id)});\n } catch (e) {\n closer.close(e);\n }\n }\n\n await closer.connected;\n return sink;\n}\n\nfunction extractStringifiedMessage(frame: string, id: number): string {\n const prefix = `{\"id\":${id},\"msg\":`;\n assert(\n frame.startsWith(prefix) && frame.endsWith('}'),\n `invalid streamed message frame`,\n );\n return frame.slice(prefix.length, -1);\n}\n\nclass WebSocketCloser {\n readonly #lc: LogContext;\n readonly #ws: WebSocket;\n readonly #closeStream: () => void;\n readonly #messageHandler: ((e: MessageEvent) => void | undefined) | null;\n readonly #connected = resolver();\n\n get connected(): Promise<void> {\n return this.#connected.promise;\n }\n\n static forSource<T>(lc: LogContext, ws: WebSocket, stream: Source<T>) {\n // If the websocket is closed, call cancel() to notify the Source of\n // any unconsumed messages.\n return new WebSocketCloser(lc, ws, () => stream.cancel());\n }\n\n static forSink<T, Input>(\n lc: LogContext,\n ws: WebSocket,\n stream: Subscription<T, Input>,\n messageHandler: (e: MessageEvent) => void | undefined,\n ) {\n // If the websocket is closed, call end() to allow the downstream Sink\n // to process any pending messages before closing the stream.\n return new WebSocketCloser(lc, ws, () => stream.end(), messageHandler);\n }\n\n private constructor(\n lc: LogContext,\n ws: WebSocket,\n closeStream: () => void,\n messageHandler?: (e: MessageEvent) => void | undefined,\n ) {\n this.#lc = lc;\n this.#ws = ws;\n this.#closeStream = closeStream;\n this.#messageHandler = messageHandler ?? null;\n\n ws.addEventListener('open', this.#handleOpen);\n ws.addEventListener('close', this.#handleClose);\n ws.addEventListener('error', this.#handleError);\n if (this.#messageHandler) {\n ws.addEventListener('message', this.#messageHandler);\n }\n\n switch (ws.readyState) {\n case ws.CONNECTING:\n break; // expected for new connections. resolve or reject in handlers.\n case ws.OPEN:\n this.#connected.resolve();\n break;\n default:\n this.#connected.reject(\n new Error(`websocket already in state ${ws.readyState}`),\n );\n break;\n }\n }\n\n get #conn(): string {\n return 'connection' + (this.#ws.url ? ` to ${this.#ws.url}` : '');\n }\n\n #handleOpen = () => {\n this.#lc.info?.(`${this.#conn} established`);\n this.#connected.resolve();\n };\n\n #handleClose = (e: CloseEvent) => {\n const {code, reason, wasClean} = e;\n this.#lc.info?.(`${this.#conn} closed`, {\n code,\n reason,\n wasClean,\n });\n this.close();\n this.#connected.reject(`${this.#conn} closed with code ${code}`);\n };\n\n #handleError = ({message, error}: ErrorEvent) => {\n if (this.#ws.readyState === this.#ws.OPEN) {\n this.#lc.error?.(`error in ${this.#conn}`, message, error);\n }\n this.#connected.reject(error);\n };\n\n close(err?: unknown) {\n if (err) {\n this.#lc.error?.(`closing stream with error`, err);\n }\n this.#closeStream();\n if (!this.closed()) {\n this.#ws.close();\n }\n }\n\n closed() {\n return (\n this.#ws.readyState === this.#ws.CLOSED ||\n this.#ws.readyState === this.#ws.CLOSING\n );\n }\n}\n"],"mappings":";;;;;;;;;;AA6BA,IAAM,mBAAmB;;;;;AAiDzB,SAAgB,OACd,IACA,IACA,UACA,aAA2B,CAAC,GAC5B,YAAyB,CAAC,GAC1B,gBAA+B,CAAC,GACc;CAC9C,MAAM,WAAW,GAAG,OAAO;CAC3B,SAAS,MAAM,KAAe;EAC5B,IAAI,GAAG,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,SACtD,IAAI,KACF,eAAe,IAAI,IAAI,GAAG;OACrB;GACL,GAAG,OAAO,yBAAyB,UAAU;GAC7C,GAAG,MAAM;EACX;CAEJ;CAEA,MAAM,WAAW,aAAa,OAAW;EACvC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,UAAU,UAAU,YAAY,GAAG;GACnC,MAAM,GAAG;EACX;CACF,CAAC;CACD,MAAM,YAAY,aAAa,OAAY;EACzC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,WAAW,UAAU,YAAY,GAAG;GACpC,MAAM,GAAG;EACX;CACF,CAAC;CAED,MAAM,SAAS,sBAAsB,IAAI;EACvC,GAAG;EACH,eAAe;CACjB,CAAC;CAGD,SAAS,YAAY;EAEnB,OAAO,GAAG,eAAe,GAAG,QAAQ,uBAAuB,UAAU,CAAC;EAEtE,SACE,SAAS,KAAK,SAAS,GACvB,IAAI,UAAU;GACZ,YAAY;GACZ,YAAY,KAAK,WAAW,aAC1B,SAAS,MAAM,WAAW,UAAU,GAAG,CAAC;EAC5C,CAAC,GACD,SACA,QAAQ,MAAM,UAAU,KAAK,GAAG,IAAI,UAAU,OAAO,CACvD;CACF;CAEA,IAAI,GAAG,eAAe,GAAG,YACvB,GAAG,GAAG,cAAc;EAClB,GAAG,OAAO,gBAAgB,UAAU;EACpC,UAAU;CACZ,CAAC;MAED,UAAU;CAIZ,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,UAAS;GAEd,OAAO,MADM,WAAW,MAAM,MAAM,SAAS,CAC9B,GAAM,UAAU,aAAa;EAC9C;CACF,CAAC;CAED,qBAAqB,IAAI,IAAI,gBAAgB;CAE7C,OAAO;EAAC;EAAW;CAAQ;AAC7B;AASA,SAAgB,KAAQ,EAAC,QAAQ,MAAM,OAAO,kBAAiC;CAC7E,mBAAmB;CACnB,OAAO,kBAAkB,GAAG,qCAAqC;CACjE,MAAM,UAA8B,CAAC;CAErC,SACE,QACA,IAAI,SAAS;EACX,eAAe;EACf,QAAQ,OAAO,WAAW,aAAa;GACrC,IAAI;GACJ,IAAI;IACF,KAAK,MAAM,MAAM,KAAK,OAAO,MAAM;KACjC,SAAS;KACT;IACF;GACF,SAAS,KAAK;IACZ,SAAS,YAAY,GAAG,CAAC;IACzB;GACF;GAMA,MAAM,EAAC,WAAU,KAAK,KAAK,GAAG;GAC9B,QAAQ,KAAK,MAAM;GACnB,OAAY,WAAW,QAAQ,MAAM,CAAC;GAEtC,IAAI,QAAQ,UAAU,gBAEpB,SAAS;QAGT,QAAQ,GAAG,WACH,SAAS,IACf,QAAO,SAAS,YAAY,GAAG,CAAC,CAClC;EAEJ;EACA,UAAU,KAAK,aAAa;GAC1B,IAAI,KACF,KAAK,KAAK,YAAY,GAAG,CAAC;GAG5B,SAAS;EACX;EACA,QAAO,aAAY;GACjB,KAAK,OAAO;GACZ,SAAS;EACX;CACF,CAAC,IACD,QAAQ,MAAM,KAAK,KAAK,GAAG,IAAI,KAAK,OAAO,CAC7C;AACF;AAEA,SAAS,YAAY,KAAc;CACjC,OAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAC3D;AAEA,IAAM,YAAY,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC;AAU5C,SAAgB,UACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,MAAM,WAAW,SAAS;AACjE;;;;AAKA,SAAgB,qBACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,OAAM,SAAQ,IAAI;AACzD;AAEA,eAAe,kBACb,IACA,QACA,MACA,WACe;CACf,qBAAqB,IAAI,MAAM,gBAAgB;CAE/C,MAAM,SAAS,gBAAgB,UAAU,IAAI,MAAM,MAAM;CAEzD,MAAM,OAAO,IAAI,MAAW;CAC5B,KAAK,iBAAiB,YAAY,EAAC,WAAU;EAC3C,IAAI;GACF,IAAI,OAAO,SAAS,UAClB,MAAM,IAAI,MAAM,yBAAyB;GAE3C,KAAK,QAAQ,MAAQ,KAAK,MAAM,IAAI,GAAG,SAAS,CAAC;EACnD,SAAS,GAAG;GACV,GAAG,QAAQ,qBAAqB,CAAC;GACjC,OAAO,MAAM,CAAC;EAChB;CACF,CAAC;CAED,IAAI;EACF,IAAI,SAAS;EACb,MAAM,EAAC,aAAY;EACnB,IAAI,UAAU;GACZ,GAAG,QAAQ,mCAAmC;GAC9C,WAAW,MAAM,EAAC,OAAO,KAAK,cAAa,UAAU;IACnD,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,CAAM,YAAY;KAChB,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;KAEjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;KAEpD,SAAS;IACX,GAAG;GACL;EACF,OAAO;GACL,GAAG,QAAQ,qCAAqC;GAChD,WAAW,MAAM,OAAO,QAAQ;IAC9B,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;IACjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;GAEtD;EACF;EACA,OAAO,MAAM;CACf,SAAS,GAAG;EACV,OAAO,MAAM,CAAC;CAChB;AACF;AAEA,SAAgB,SACd,IACA,QACA,QACoB;CACpB,OAAO,iBAAiB,IAAI,QAAQ,SAAQ,SAAQ,IAAI;AAC1D;;;;;AAMA,SAAgB,oBACd,IACA,QACA,QACgC;CAChC,OAAO,iBAAiB,IAAI,QAAQ,SAAS,MAAM,OAAO,QAAQ;EAChE;EACA,MAAM,0BAA0B,OAAO,EAAE;CAC3C,EAAE;AACJ;AAEA,eAAe,iBACb,IACA,QACA,QACA,WACsB;CACtB,uBAAuB,IAAI,QAAQ,gBAAgB;CAEnD,MAAM,iBAAiB,eAAE,OAAO;EAC9B,KAAK;EACL,IAAI,eAAE,OAAO;CACf,CAAC;CAED,MAAM,OAAmD,IAAI,aAI3D;EACE,WAAW,EAAC,SAAQ,OAAO,KAAK,KAAK,UAAU,EAAC,KAAK,GAAE,CAAe,CAAC;EACvE,eAAe,OAAO,MAAM;CAC9B,IACC,EAAC,WAAU,IACd;CAEA,MAAM,SAAS,gBAAgB,QAAQ,IAAI,QAAQ,MAAM,aAAa;CAEtE,SAAS,cAAc,OAAqB;EAC1C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,CAAC,KAAK,QAAQ;GAChB,GAAG,OAAO,4CAA4C,IAAI;GAC1D;EACF;EACA,IAAI;GAEF,MAAM,MAAM,MADE,WAAW,MAAM,IACX,GAAO,gBAAgB,aAAa;GAGxD,KAAK,KAAK;IAAC,IAAI,IAAI;IAAI,MAAM,UAAU,IAAI,KAAK,MAAM,IAAI,EAAE;GAAC,CAAC;EAChE,SAAS,GAAG;GACV,OAAO,MAAM,CAAC;EAChB;CACF;CAEA,MAAM,OAAO;CACb,OAAO;AACT;AAEA,SAAS,0BAA0B,OAAe,IAAoB;CACpE,MAAM,SAAS,SAAS,GAAG;CAC3B,OACE,MAAM,WAAW,MAAM,KAAK,MAAM,SAAS,GAAG,GAC9C,gCACF;CACA,OAAO,MAAM,MAAM,OAAO,QAAQ,EAAE;AACtC;AAEA,IAAM,kBAAN,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA,aAAsB,SAAS;CAE/B,IAAI,YAA2B;EAC7B,OAAO,KAAKI,WAAW;CACzB;CAEA,OAAO,UAAa,IAAgB,IAAe,QAAmB;EAGpE,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,OAAO,CAAC;CAC1D;CAEA,OAAO,QACL,IACA,IACA,QACA,gBACA;EAGA,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,IAAI,GAAG,cAAc;CACvE;CAEA,YACE,IACA,IACA,aACA,gBACA;EACA,KAAKJ,MAAM;EACX,KAAKC,MAAM;EACX,KAAKC,eAAe;EACpB,KAAKC,kBAAkB,kBAAkB;EAEzC,GAAG,iBAAiB,QAAQ,KAAKE,WAAW;EAC5C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,IAAI,KAAKJ,iBACP,GAAG,iBAAiB,WAAW,KAAKA,eAAe;EAGrD,QAAQ,GAAG,YAAX;GACE,KAAK,GAAG,YACN;GACF,KAAK,GAAG;IACN,KAAKC,WAAW,QAAQ;IACxB;GACF;IACE,KAAKA,WAAW,uBACd,IAAI,MAAM,8BAA8B,GAAG,YAAY,CACzD;IACA;EACJ;CACF;CAEA,IAAII,QAAgB;EAClB,OAAO,gBAAgB,KAAKP,IAAI,MAAM,OAAO,KAAKA,IAAI,QAAQ;CAChE;CAEA,oBAAoB;EAClB,KAAKD,IAAI,OAAO,GAAG,KAAKQ,MAAM,aAAa;EAC3C,KAAKJ,WAAW,QAAQ;CAC1B;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,KAAKJ,IAAI,OAAO,GAAG,KAAKQ,MAAM,UAAU;GACtC;GACA;GACA;EACF,CAAC;EACD,KAAK,MAAM;EACX,KAAKJ,WAAW,OAAO,GAAG,KAAKI,MAAM,oBAAoB,MAAM;CACjE;CAEA,gBAAgB,EAAC,SAAS,YAAuB;EAC/C,IAAI,KAAKP,IAAI,eAAe,KAAKA,IAAI,MACnC,KAAKD,IAAI,QAAQ,YAAY,KAAKQ,SAAS,SAAS,KAAK;EAE3D,KAAKJ,WAAW,OAAO,KAAK;CAC9B;CAEA,MAAM,KAAe;EACnB,IAAI,KACF,KAAKJ,IAAI,QAAQ,6BAA6B,GAAG;EAEnD,KAAKE,aAAa;EAClB,IAAI,CAAC,KAAK,OAAO,GACf,KAAKD,IAAI,MAAM;CAEnB;CAEA,SAAS;EACP,OACE,KAAKA,IAAI,eAAe,KAAKA,IAAI,UACjC,KAAKA,IAAI,eAAe,KAAKA,IAAI;CAErC;AACF"}
|
|
1
|
+
{"version":3,"file":"streams.js","names":["#lc","#ws","#closeStream","#messageHandler","#connected","#handleOpen","#handleClose","#handleError","#conn"],"sources":["../../../../../zero-cache/src/types/streams.ts"],"sourcesContent":["import {\n pipeline,\n Readable,\n Transform,\n Writable,\n type DuplexOptions,\n} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n createWebSocketStream,\n type CloseEvent,\n type ErrorEvent,\n type MessageEvent,\n type WebSocket,\n} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {BigIntJSON, type JSONValue} from '../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Subscription, type Options} from './subscription.ts';\nimport {\n closeWithError,\n expectPingsForLiveness,\n sendPingsForLiveness,\n} from './ws.ts';\n\n// Consistent with Postgres keepalives, and shorter than the\n// commonly used default idle timeout of 1 minute.\nconst PING_INTERVAL_MS = 30_000;\n\nexport type Source<T> = AsyncIterable<T> & {\n /**\n * Immediately terminates all current iterations (i.e. {@link AsyncIterator.next next()})\n * will return `{value: undefined, done: true}`), and prevents any subsequent iterations\n * from yielding any values.\n *\n * @param err Terminate the iteration by throwing the `err` instead.\n */\n cancel: (err?: Error) => void;\n\n /**\n * An AbortSignal that can be used to listen for termination or\n * race a short-lived promise against the termination of the iterable\n * (via `promiseOrAbort()`)\n */\n readonly signal: AbortSignal;\n\n /**\n * The presence of a `pipeline` iterable allows the usual \"consumed-on-iterate\" semantics\n * to be overridden.\n *\n * This is suitable for transport layers that serialize messages across processes, such\n * as the {@link streamOut()} method; pipelining allows the transport to send messages\n * as they arrive without waiting for the previous message to be acked, streaming\n * them to the receiving process where they are presumably queued and processed without\n * a per-message ack delay. The receiving end of the transport then responds with acks\n * asynchronously as the receiving end processes the messages.\n */\n pipeline?: AsyncIterable<{value: T; consumed: () => void}> | undefined;\n};\n\nexport type Sink<T> = {\n push(message: T): void;\n};\n\n/**\n * Back-pressure-aware transformation of a WebSocket into\n * upstream and downstream {@link Subscription} objects.\n */\n// TODO: Change {@link streamIn} and {@link streamOut} to use this\n// under the covers so that internal communication is also\n// responsive to backpressure.\nexport function stream<In extends JSONValue, Out extends JSONValue>(\n lc: LogContext,\n ws: WebSocket,\n inSchema: v.Type<In>,\n outOptions: Options<Out> = {},\n inOptions: Options<In> = {},\n streamOptions: DuplexOptions = {},\n): {outstream: Sink<Out>; instream: Source<In>} {\n const endpoint = ws.url ?? 'client';\n function close(err?: unknown) {\n if (ws.readyState !== ws.CLOSED && ws.readyState !== ws.CLOSING) {\n if (err) {\n closeWithError(lc, ws, err);\n } else {\n lc.info?.(`closing connection to ${endpoint}`);\n ws.close();\n }\n }\n }\n\n const instream = Subscription.create<In>({\n ...inOptions,\n cleanup: (unconsumed, err) => {\n inOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n const outstream = Subscription.create<Out>({\n ...outOptions,\n cleanup: (unconsumed, err) => {\n outOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n\n const duplex = createWebSocketStream(ws, {\n ...streamOptions,\n decodeStrings: false,\n });\n\n // Outgoing transform.\n function streamOut() {\n // Mainly used for verifying that back-pressure kicks in tests.\n duplex.on('drain', () => lc.debug?.(`drained messages to ${endpoint}`));\n\n pipeline(\n Readable.from(outstream),\n new Transform({\n objectMode: true,\n transform: (msg, _encoding, callback) =>\n callback(null, BigIntJSON.stringify(msg)),\n }),\n duplex,\n err => (err ? outstream.fail(err) : outstream.cancel()),\n );\n }\n\n if (ws.readyState === ws.CONNECTING) {\n ws.on('open', () => {\n lc.info?.(`connected to ${endpoint}`);\n streamOut();\n });\n } else {\n streamOut();\n }\n\n // Incoming transform.\n pipe({\n source: duplex,\n sink: instream,\n parse: chunk => {\n const json = BigIntJSON.parse(chunk.toString());\n return v.parse(json, inSchema, 'passthrough');\n },\n });\n\n sendPingsForLiveness(lc, ws, PING_INTERVAL_MS);\n\n return {outstream, instream};\n}\n\ntype PipeOptions<T> = {\n source: Readable;\n sink: Subscription<T>;\n parse: (buffer: Buffer) => T | null;\n bufferMessages?: number;\n};\n\nexport function pipe<T>({source, sink, parse, bufferMessages}: PipeOptions<T>) {\n bufferMessages ??= 0;\n assert(bufferMessages >= 0, 'bufferMessages must be non-negative');\n const pending: Promise<unknown>[] = [];\n\n pipeline(\n source,\n new Writable({\n decodeStrings: false,\n write: (chunk, _encoding, callback) => {\n let msg: T | null;\n try {\n if ((msg = parse(chunk)) === null) {\n callback();\n return;\n }\n } catch (err) {\n callback(ensureError(err));\n return;\n }\n // Inbound backpressure is exerted by unconsumed messages in the\n // subscription. A buffer can be used to allow messages to queue up in\n // in the Subscription object, which allows the consumer to \"peek\" at\n // whether there are more messages immediately available\n // (via {@link Subscription.queued}.\n const {result} = sink.push(msg);\n pending.push(result);\n void result.then(() => pending.shift());\n\n if (pending.length <= bufferMessages) {\n // immediately allow more messages\n callback();\n } else {\n // wait for the oldest result in the pending queue\n pending[0].then(\n () => callback(),\n err => callback(ensureError(err)),\n );\n }\n },\n destroy: (err, callback) => {\n if (err) {\n sink.fail(ensureError(err));\n }\n // Otherwise, final will handle the cancel.\n callback();\n },\n final: callback => {\n sink.cancel();\n callback();\n },\n }),\n err => (err ? sink.fail(err) : sink.cancel()),\n );\n}\n\nfunction ensureError(err: unknown) {\n return err instanceof Error ? err : new Error(String(err));\n}\n\nconst ackSchema = v.object({ack: v.number()});\n\ntype Ack = v.Infer<typeof ackSchema>;\n\n/** A parsed JSON value paired with the exact source text it was parsed from. */\nexport type ParsedJSON<T> = {\n data: T;\n json: string;\n};\n\nexport function streamOut<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, BigIntJSON.stringify);\n}\n\n/**\n * Streams out a `Source` for which messages are already stringified JSON.\n */\nexport function streamOutStringified(\n lc: LogContext,\n source: Source<string>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, json => json);\n}\n\nasync function streamOutInternal<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n stringify: (payload: T) => string,\n): Promise<void> {\n sendPingsForLiveness(lc, sink, PING_INTERVAL_MS);\n\n const closer = WebSocketCloser.forSource(lc, sink, source);\n\n const acks = new Queue<Ack>();\n sink.addEventListener('message', ({data}) => {\n try {\n if (typeof data !== 'string') {\n throw new Error('Expected string message');\n }\n acks.enqueue(v.parse(JSON.parse(data), ackSchema));\n } catch (e) {\n lc.error?.(`error parsing ack`, e);\n closer.close(e);\n }\n });\n\n try {\n let nextID = 0;\n const {pipeline} = source;\n if (pipeline) {\n lc.debug?.(`started pipelined outbound stream`);\n for await (const {value: msg, consumed} of pipeline) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`pipelining`, data);\n sink.send(data);\n\n void (async () => {\n const {ack} = await acks.dequeue();\n // lc.debug?.(`received ack`, ack);\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n consumed();\n })();\n }\n } else {\n lc.debug?.(`started synchronous outbound stream`);\n for await (const msg of source) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`sending`, data);\n sink.send(data);\n\n const {ack} = await acks.dequeue();\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n }\n }\n closer.close();\n } catch (e) {\n closer.close(e);\n }\n}\n\nexport function streamIn<T extends JSONValue>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n): Promise<Source<T>> {\n return streamInInternal(lc, source, schema, data => data);\n}\n\n/**\n * Streams in messages sent by {@link streamOutStringified}, preserving the\n * exact application-level JSON alongside its parsed and validated value.\n */\nexport function streamInStringified<T extends JSONValue>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n): Promise<Source<ParsedJSON<T>>> {\n return streamInInternal(lc, source, schema, (data, frame, id) => ({\n data,\n json: extractStringifiedMessage(frame, id),\n }));\n}\n\nasync function streamInInternal<T extends JSONValue, Out>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n transform: (data: T, frame: string, id: number) => Out,\n): Promise<Source<Out>> {\n expectPingsForLiveness(lc, source, PING_INTERVAL_MS);\n\n const streamedSchema = v.object({\n msg: schema,\n id: v.number(),\n });\n\n const sink: Subscription<Out, {id: number; data: Out}> = new Subscription<\n Out,\n {id: number; data: Out}\n >(\n {\n consumed: ({id}) => source.send(JSON.stringify({ack: id} satisfies Ack)),\n cleanup: () => closer.close(),\n },\n ({data}) => data,\n );\n\n const closer = WebSocketCloser.forSink(lc, source, sink, handleMessage);\n\n function handleMessage(event: MessageEvent) {\n const data = event.data.toString();\n if (!sink.active) {\n lc.warn?.('dropping ws message received after close', data);\n return;\n }\n try {\n const value = BigIntJSON.parse(data);\n const msg = v.parse(value, streamedSchema, 'passthrough');\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`received`, data);\n sink.push({id: msg.id, data: transform(msg.msg, data, msg.id)});\n } catch (e) {\n closer.close(e);\n }\n }\n\n await closer.connected;\n return sink;\n}\n\nfunction extractStringifiedMessage(frame: string, id: number): string {\n const prefix = `{\"id\":${id},\"msg\":`;\n assert(\n frame.startsWith(prefix) && frame.endsWith('}'),\n `invalid streamed message frame`,\n );\n return frame.slice(prefix.length, -1);\n}\n\nclass WebSocketCloser {\n readonly #lc: LogContext;\n readonly #ws: WebSocket;\n readonly #closeStream: () => void;\n readonly #messageHandler: ((e: MessageEvent) => void | undefined) | null;\n readonly #connected = resolver();\n\n get connected(): Promise<void> {\n return this.#connected.promise;\n }\n\n static forSource<T>(lc: LogContext, ws: WebSocket, stream: Source<T>) {\n // If the websocket is closed, call cancel() to notify the Source of\n // any unconsumed messages.\n return new WebSocketCloser(lc, ws, () => stream.cancel());\n }\n\n static forSink<T, Input>(\n lc: LogContext,\n ws: WebSocket,\n stream: Subscription<T, Input>,\n messageHandler: (e: MessageEvent) => void | undefined,\n ) {\n // If the websocket is closed, call end() to allow the downstream Sink\n // to process any pending messages before closing the stream.\n return new WebSocketCloser(lc, ws, () => stream.end(), messageHandler);\n }\n\n private constructor(\n lc: LogContext,\n ws: WebSocket,\n closeStream: () => void,\n messageHandler?: (e: MessageEvent) => void | undefined,\n ) {\n this.#lc = lc;\n this.#ws = ws;\n this.#closeStream = closeStream;\n this.#messageHandler = messageHandler ?? null;\n\n ws.addEventListener('open', this.#handleOpen);\n ws.addEventListener('close', this.#handleClose);\n ws.addEventListener('error', this.#handleError);\n if (this.#messageHandler) {\n ws.addEventListener('message', this.#messageHandler);\n }\n\n switch (ws.readyState) {\n case ws.CONNECTING:\n break; // expected for new connections. resolve or reject in handlers.\n case ws.OPEN:\n this.#connected.resolve();\n break;\n default:\n this.#connected.reject(\n new Error(`websocket already in state ${ws.readyState}`),\n );\n break;\n }\n }\n\n get #conn(): string {\n return 'connection' + (this.#ws.url ? ` to ${this.#ws.url}` : '');\n }\n\n #handleOpen = () => {\n this.#lc.info?.(`${this.#conn} established`);\n this.#connected.resolve();\n };\n\n #handleClose = (e: CloseEvent) => {\n const {code, reason, wasClean} = e;\n this.#lc.info?.(`${this.#conn} closed`, {\n code,\n reason,\n wasClean,\n });\n this.close();\n this.#connected.reject(`${this.#conn} closed with code ${code}`);\n };\n\n #handleError = ({message, error}: ErrorEvent) => {\n if (this.#ws.readyState === this.#ws.OPEN) {\n this.#lc.error?.(`error in ${this.#conn}`, message, error);\n }\n this.#connected.reject(error);\n };\n\n close(err?: unknown) {\n if (err) {\n this.#lc.error?.(`closing stream with error`, err);\n }\n this.#closeStream();\n if (!this.closed()) {\n this.#ws.close();\n }\n }\n\n closed() {\n return (\n this.#ws.readyState === this.#ws.CLOSED ||\n this.#ws.readyState === this.#ws.CLOSING\n );\n }\n}\n"],"mappings":";;;;;;;;;;AA6BA,IAAM,mBAAmB;;;;;AA4CzB,SAAgB,OACd,IACA,IACA,UACA,aAA2B,CAAC,GAC5B,YAAyB,CAAC,GAC1B,gBAA+B,CAAC,GACc;CAC9C,MAAM,WAAW,GAAG,OAAO;CAC3B,SAAS,MAAM,KAAe;EAC5B,IAAI,GAAG,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,SACtD,IAAI,KACF,eAAe,IAAI,IAAI,GAAG;OACrB;GACL,GAAG,OAAO,yBAAyB,UAAU;GAC7C,GAAG,MAAM;EACX;CAEJ;CAEA,MAAM,WAAW,aAAa,OAAW;EACvC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,UAAU,UAAU,YAAY,GAAG;GACnC,MAAM,GAAG;EACX;CACF,CAAC;CACD,MAAM,YAAY,aAAa,OAAY;EACzC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,WAAW,UAAU,YAAY,GAAG;GACpC,MAAM,GAAG;EACX;CACF,CAAC;CAED,MAAM,SAAS,sBAAsB,IAAI;EACvC,GAAG;EACH,eAAe;CACjB,CAAC;CAGD,SAAS,YAAY;EAEnB,OAAO,GAAG,eAAe,GAAG,QAAQ,uBAAuB,UAAU,CAAC;EAEtE,SACE,SAAS,KAAK,SAAS,GACvB,IAAI,UAAU;GACZ,YAAY;GACZ,YAAY,KAAK,WAAW,aAC1B,SAAS,MAAM,WAAW,UAAU,GAAG,CAAC;EAC5C,CAAC,GACD,SACA,QAAQ,MAAM,UAAU,KAAK,GAAG,IAAI,UAAU,OAAO,CACvD;CACF;CAEA,IAAI,GAAG,eAAe,GAAG,YACvB,GAAG,GAAG,cAAc;EAClB,GAAG,OAAO,gBAAgB,UAAU;EACpC,UAAU;CACZ,CAAC;MAED,UAAU;CAIZ,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,UAAS;GAEd,OAAO,MADM,WAAW,MAAM,MAAM,SAAS,CAC9B,GAAM,UAAU,aAAa;EAC9C;CACF,CAAC;CAED,qBAAqB,IAAI,IAAI,gBAAgB;CAE7C,OAAO;EAAC;EAAW;CAAQ;AAC7B;AASA,SAAgB,KAAQ,EAAC,QAAQ,MAAM,OAAO,kBAAiC;CAC7E,mBAAmB;CACnB,OAAO,kBAAkB,GAAG,qCAAqC;CACjE,MAAM,UAA8B,CAAC;CAErC,SACE,QACA,IAAI,SAAS;EACX,eAAe;EACf,QAAQ,OAAO,WAAW,aAAa;GACrC,IAAI;GACJ,IAAI;IACF,KAAK,MAAM,MAAM,KAAK,OAAO,MAAM;KACjC,SAAS;KACT;IACF;GACF,SAAS,KAAK;IACZ,SAAS,YAAY,GAAG,CAAC;IACzB;GACF;GAMA,MAAM,EAAC,WAAU,KAAK,KAAK,GAAG;GAC9B,QAAQ,KAAK,MAAM;GACnB,OAAY,WAAW,QAAQ,MAAM,CAAC;GAEtC,IAAI,QAAQ,UAAU,gBAEpB,SAAS;QAGT,QAAQ,GAAG,WACH,SAAS,IACf,QAAO,SAAS,YAAY,GAAG,CAAC,CAClC;EAEJ;EACA,UAAU,KAAK,aAAa;GAC1B,IAAI,KACF,KAAK,KAAK,YAAY,GAAG,CAAC;GAG5B,SAAS;EACX;EACA,QAAO,aAAY;GACjB,KAAK,OAAO;GACZ,SAAS;EACX;CACF,CAAC,IACD,QAAQ,MAAM,KAAK,KAAK,GAAG,IAAI,KAAK,OAAO,CAC7C;AACF;AAEA,SAAS,YAAY,KAAc;CACjC,OAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAC3D;AAEA,IAAM,YAAY,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC;AAU5C,SAAgB,UACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,MAAM,WAAW,SAAS;AACjE;;;;AAKA,SAAgB,qBACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,OAAM,SAAQ,IAAI;AACzD;AAEA,eAAe,kBACb,IACA,QACA,MACA,WACe;CACf,qBAAqB,IAAI,MAAM,gBAAgB;CAE/C,MAAM,SAAS,gBAAgB,UAAU,IAAI,MAAM,MAAM;CAEzD,MAAM,OAAO,IAAI,MAAW;CAC5B,KAAK,iBAAiB,YAAY,EAAC,WAAU;EAC3C,IAAI;GACF,IAAI,OAAO,SAAS,UAClB,MAAM,IAAI,MAAM,yBAAyB;GAE3C,KAAK,QAAQ,MAAQ,KAAK,MAAM,IAAI,GAAG,SAAS,CAAC;EACnD,SAAS,GAAG;GACV,GAAG,QAAQ,qBAAqB,CAAC;GACjC,OAAO,MAAM,CAAC;EAChB;CACF,CAAC;CAED,IAAI;EACF,IAAI,SAAS;EACb,MAAM,EAAC,aAAY;EACnB,IAAI,UAAU;GACZ,GAAG,QAAQ,mCAAmC;GAC9C,WAAW,MAAM,EAAC,OAAO,KAAK,cAAa,UAAU;IACnD,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,CAAM,YAAY;KAChB,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;KAEjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;KAEpD,SAAS;IACX,GAAG;GACL;EACF,OAAO;GACL,GAAG,QAAQ,qCAAqC;GAChD,WAAW,MAAM,OAAO,QAAQ;IAC9B,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;IACjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;GAEtD;EACF;EACA,OAAO,MAAM;CACf,SAAS,GAAG;EACV,OAAO,MAAM,CAAC;CAChB;AACF;AAEA,SAAgB,SACd,IACA,QACA,QACoB;CACpB,OAAO,iBAAiB,IAAI,QAAQ,SAAQ,SAAQ,IAAI;AAC1D;;;;;AAMA,SAAgB,oBACd,IACA,QACA,QACgC;CAChC,OAAO,iBAAiB,IAAI,QAAQ,SAAS,MAAM,OAAO,QAAQ;EAChE;EACA,MAAM,0BAA0B,OAAO,EAAE;CAC3C,EAAE;AACJ;AAEA,eAAe,iBACb,IACA,QACA,QACA,WACsB;CACtB,uBAAuB,IAAI,QAAQ,gBAAgB;CAEnD,MAAM,iBAAiB,eAAE,OAAO;EAC9B,KAAK;EACL,IAAI,eAAE,OAAO;CACf,CAAC;CAED,MAAM,OAAmD,IAAI,aAI3D;EACE,WAAW,EAAC,SAAQ,OAAO,KAAK,KAAK,UAAU,EAAC,KAAK,GAAE,CAAe,CAAC;EACvE,eAAe,OAAO,MAAM;CAC9B,IACC,EAAC,WAAU,IACd;CAEA,MAAM,SAAS,gBAAgB,QAAQ,IAAI,QAAQ,MAAM,aAAa;CAEtE,SAAS,cAAc,OAAqB;EAC1C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,CAAC,KAAK,QAAQ;GAChB,GAAG,OAAO,4CAA4C,IAAI;GAC1D;EACF;EACA,IAAI;GAEF,MAAM,MAAM,MADE,WAAW,MAAM,IACX,GAAO,gBAAgB,aAAa;GAGxD,KAAK,KAAK;IAAC,IAAI,IAAI;IAAI,MAAM,UAAU,IAAI,KAAK,MAAM,IAAI,EAAE;GAAC,CAAC;EAChE,SAAS,GAAG;GACV,OAAO,MAAM,CAAC;EAChB;CACF;CAEA,MAAM,OAAO;CACb,OAAO;AACT;AAEA,SAAS,0BAA0B,OAAe,IAAoB;CACpE,MAAM,SAAS,SAAS,GAAG;CAC3B,OACE,MAAM,WAAW,MAAM,KAAK,MAAM,SAAS,GAAG,GAC9C,gCACF;CACA,OAAO,MAAM,MAAM,OAAO,QAAQ,EAAE;AACtC;AAEA,IAAM,kBAAN,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA,aAAsB,SAAS;CAE/B,IAAI,YAA2B;EAC7B,OAAO,KAAKI,WAAW;CACzB;CAEA,OAAO,UAAa,IAAgB,IAAe,QAAmB;EAGpE,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,OAAO,CAAC;CAC1D;CAEA,OAAO,QACL,IACA,IACA,QACA,gBACA;EAGA,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,IAAI,GAAG,cAAc;CACvE;CAEA,YACE,IACA,IACA,aACA,gBACA;EACA,KAAKJ,MAAM;EACX,KAAKC,MAAM;EACX,KAAKC,eAAe;EACpB,KAAKC,kBAAkB,kBAAkB;EAEzC,GAAG,iBAAiB,QAAQ,KAAKE,WAAW;EAC5C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,IAAI,KAAKJ,iBACP,GAAG,iBAAiB,WAAW,KAAKA,eAAe;EAGrD,QAAQ,GAAG,YAAX;GACE,KAAK,GAAG,YACN;GACF,KAAK,GAAG;IACN,KAAKC,WAAW,QAAQ;IACxB;GACF;IACE,KAAKA,WAAW,uBACd,IAAI,MAAM,8BAA8B,GAAG,YAAY,CACzD;IACA;EACJ;CACF;CAEA,IAAII,QAAgB;EAClB,OAAO,gBAAgB,KAAKP,IAAI,MAAM,OAAO,KAAKA,IAAI,QAAQ;CAChE;CAEA,oBAAoB;EAClB,KAAKD,IAAI,OAAO,GAAG,KAAKQ,MAAM,aAAa;EAC3C,KAAKJ,WAAW,QAAQ;CAC1B;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,KAAKJ,IAAI,OAAO,GAAG,KAAKQ,MAAM,UAAU;GACtC;GACA;GACA;EACF,CAAC;EACD,KAAK,MAAM;EACX,KAAKJ,WAAW,OAAO,GAAG,KAAKI,MAAM,oBAAoB,MAAM;CACjE;CAEA,gBAAgB,EAAC,SAAS,YAAuB;EAC/C,IAAI,KAAKP,IAAI,eAAe,KAAKA,IAAI,MACnC,KAAKD,IAAI,QAAQ,YAAY,KAAKQ,SAAS,SAAS,KAAK;EAE3D,KAAKJ,WAAW,OAAO,KAAK;CAC9B;CAEA,MAAM,KAAe;EACnB,IAAI,KACF,KAAKJ,IAAI,QAAQ,6BAA6B,GAAG;EAEnD,KAAKE,aAAa;EAClB,IAAI,CAAC,KAAK,OAAO,GACf,KAAKD,IAAI,MAAM;CAEnB;CAEA,SAAS;EACP,OACE,KAAKA,IAAI,eAAe,KAAKA,IAAI,UACjC,KAAKA,IAAI,eAAe,KAAKA,IAAI;CAErC;AACF"}
|