@rocicorp/zero 1.11.0-canary.40 → 1.11.0-canary.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/replicache/src/deleted-clients.d.ts +2 -2
- package/out/replicache/src/deleted-clients.js +2 -2
- package/out/replicache/src/deleted-clients.js.map +1 -1
- package/out/replicache/src/persist/client-group-gc.d.ts +1 -1
- package/out/replicache/src/persist/client-group-gc.js +1 -1
- package/out/replicache/src/persist/client-group-gc.js.map +1 -1
- package/out/replicache/src/replicache-options.d.ts +1 -1
- package/out/shared/src/browser-env.d.ts +1 -1
- package/out/shared/src/browser-env.js.map +1 -1
- package/out/shared/src/set-utils.d.ts +1 -1
- package/out/shared/src/set-utils.js +1 -1
- package/out/shared/src/set-utils.js.map +1 -1
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
- package/out/zero-cache/src/config/normalize.js +1 -0
- package/out/zero-cache/src/config/normalize.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +2 -1
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +6 -9
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +2 -0
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js +8 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +16 -5
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +20 -12
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts +21 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +73 -38
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +13 -3
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +7 -4
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/types/pg-patch.d.ts +21 -0
- package/out/zero-cache/src/types/pg-patch.d.ts.map +1 -0
- package/out/zero-cache/src/types/pg-patch.js +29 -0
- package/out/zero-cache/src/types/pg-patch.js.map +1 -0
- package/out/zero-client/src/client/connection-manager.js +1 -1
- package/out/zero-client/src/client/connection-manager.js.map +1 -1
- package/out/zero-client/src/client/custom.d.ts +1 -1
- package/out/zero-client/src/client/custom.js +1 -1
- package/out/zero-client/src/client/custom.js.map +1 -1
- package/out/zero-client/src/client/options.d.ts +6 -6
- package/out/zero-client/src/client/options.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero.d.ts +1 -1
- package/out/zero-client/src/client/zero.js +1 -1
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-protocol/src/inspect-up.d.ts +3 -3
- package/out/zero-protocol/src/inspect-up.d.ts.map +1 -1
- package/out/zero-protocol/src/inspect-up.js +2 -2
- package/out/zero-protocol/src/inspect-up.js.map +1 -1
- package/out/zero-react/src/use-zero-online.d.ts +1 -1
- package/out/zero-react/src/use-zero-online.js +1 -1
- package/out/zero-react/src/use-zero-online.js.map +1 -1
- package/out/zero-react/src/zero-provider.d.ts +1 -1
- package/out/zero-react/src/zero-provider.js +1 -1
- package/out/zero-react/src/zero-provider.js.map +1 -1
- package/out/zero-server/src/closable-db-transaction.d.ts +1 -1
- package/out/zero-server/src/closable-db-transaction.js +1 -1
- package/out/zero-server/src/closable-db-transaction.js.map +1 -1
- package/out/zero-server/src/custom.d.ts +1 -1
- package/out/zero-server/src/custom.js +1 -1
- package/out/zero-server/src/custom.js.map +1 -1
- package/out/zero-server/src/process-mutations.d.ts +3 -3
- package/out/zero-server/src/process-mutations.js +1 -1
- package/out/zero-server/src/process-mutations.js.map +1 -1
- package/out/zero-server/src/push-processor.d.ts +1 -1
- package/out/zero-server/src/push-processor.js +1 -1
- package/out/zero-server/src/push-processor.js.map +1 -1
- package/out/zero-server/src/queries/process-queries.d.ts +3 -3
- package/out/zero-server/src/queries/process-queries.js +2 -2
- package/out/zero-server/src/queries/process-queries.js.map +1 -1
- package/out/zero-server/src/schema.d.ts +1 -1
- package/out/zero-server/src/schema.js +1 -1
- package/out/zero-server/src/schema.js.map +1 -1
- package/out/zero-server/src/zql-database.d.ts +2 -2
- package/out/zero-server/src/zql-database.js +2 -2
- package/out/zero-server/src/zql-database.js.map +1 -1
- package/out/zero-solid/src/use-query.d.ts +2 -2
- package/out/zero-solid/src/use-query.js +1 -1
- package/out/zero-solid/src/use-query.js.map +1 -1
- package/out/zero-solid/src/use-zero-online.d.ts +1 -1
- package/out/zero-solid/src/use-zero-online.js +1 -1
- package/out/zero-solid/src/use-zero-online.js.map +1 -1
- package/out/zero-solid/src/use-zero.d.ts +2 -2
- package/out/zero-solid/src/use-zero.js +2 -2
- package/out/zero-solid/src/use-zero.js.map +1 -1
- package/out/zql/src/ivm/change.d.ts +3 -3
- package/out/zql/src/ivm/change.js.map +1 -1
- package/out/zql/src/ivm/data.d.ts +1 -1
- package/out/zql/src/ivm/data.js.map +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.d.ts +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js.map +1 -1
- package/out/zql/src/mutate/custom.d.ts +3 -3
- package/out/zql/src/mutate/custom.js.map +1 -1
- package/out/zql/src/query/create-builder.d.ts +1 -1
- package/out/zql/src/query/create-builder.d.ts.map +1 -1
- package/out/zql/src/query/create-builder.js +1 -1
- package/out/zql/src/query/create-builder.js.map +1 -1
- package/out/zql/src/query/named.d.ts +2 -2
- package/out/zql/src/query/named.js +2 -2
- package/out/zql/src/query/named.js.map +1 -1
- package/out/zql/src/query/query-impl.js +3 -3
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query-internals.d.ts +5 -5
- package/out/zql/src/query/query-internals.js.map +1 -1
- package/out/zql/src/query/query-transitions.d.ts +5 -5
- package/out/zql/src/query/query-transitions.js +2 -2
- package/out/zql/src/query/query-transitions.js.map +1 -1
- package/out/zql/src/query/query.d.ts +8 -8
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#makeConnectionPool","#db","#inserter","#purger","#replicaVersion","#onCommitted","#onFatal","#queue","#backPressureThresholdBytes","#statementTimeoutMs","#changeLogBatchSize","#drainTimeoutMs","#progressMonitor","#withTimeout","#ownerAddress","#pendingOwnership","#tryAssumeOwnership","#cdc","#approximateQueuedBytes","#running","#readyForMore","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#ownershipAssumed","#processChange","#maybeReleaseBackPressure","#flushChangeLog","#trackBackfillMetadata","#catchup","#cookieStmts","#upsertTableMetadataStmt","#upsertColumnBackfillStmt","#tx","#released","#failureThresholdMs","#onFailure","#taskUpdates","#timer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {getHeapStatistics} from 'node:v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {\n BigIntJSON,\n type JSONObject,\n} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {sharedSnapshot, TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {orTimeout} from '../../types/timeout.ts';\nimport {\n backfillRequestSchema,\n isDataChange,\n isSchemaChange,\n type BackfillID,\n type BackfillRequest,\n type Change,\n type DataChange,\n type Identifier,\n type SchemaChange,\n type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {\n type ChangeStreamData,\n type Commit,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {UpstreamStatusMessage} from '../change-source/protocol/current/status.ts';\nimport {\n cookieOps,\n type BackfillCookie,\n type CookieOp,\n type CookieSet,\n type TableMetadataCookie,\n} from '../replicator/change-log-cookies.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport {\n reconstructWatermarkedChange,\n serializeChangeStreamDataWithChange,\n type ChangeLogEntry,\n type SerializedChangeStreamData,\n} from './change-log-codec.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n AutoResetSignal,\n markResetRequired,\n reinitializeChangeLog,\n type BackfillingColumn,\n type ChangeLogReinitialization,\n type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\n/**\n * Factory for creating dynamic connection pools used by the Storer to\n * isolate concurrent transactions into separate pg clients.\n */\nexport type PostgresDBProvider = (\n applicationName: string,\n maxConns: number,\n) => PostgresDB;\n\ntype SubscriberAndMode = {\n subscriber: Subscriber;\n mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n | [\n 'change',\n watermark: string,\n json: string,\n storedChange: string,\n orig: Exclude<Change, DataChange> | null, // null for DataChanges\n ]\n | ['ready', callback: () => void]\n | ['subscriber', SubscriberAndMode]\n | ['status', watermark: string]\n | ['abort']\n | 'stop';\n\n/**\n * A transaction that is not stored because it was already stored by the\n * previous owner of the change log (see {@link Storer.assumeOwnershipAtHead}).\n */\ntype SkippedTransaction = {\n skipped: true;\n ack: boolean;\n};\n\ntype PendingTransaction = {\n skipped?: undefined;\n pool: TransactionPool;\n preCommitWatermark: string;\n pos: number;\n startingReplicationState: Promise<ReplicationOwner>;\n ack: boolean;\n // changeLog rows buffered for the next multi-row INSERT flush.\n batch: ChangeLogRow[];\n // The most recently issued flush (or metadata) process, awaited to bound\n // pipeline depth and to order the commit-time replicationState update.\n lastFlush: Promise<unknown> | undefined;\n // Set if ownership of the change log is assumed in this transaction.\n ownershipAssumed: AssumedOwnership | undefined;\n};\n\n/** The state of the change log at which ownership was assumed. */\ntype AssumedOwnership = {\n /** The last watermark stored by the previous owner. */\n head: string;\n /** Whether the change log has pending backfills. */\n backfilling: boolean;\n};\n\nexport type OwnershipTakeover = {\n /**\n * The lock that prevents the previous owner from purging the change log\n * (from which this change-streamer's replica was restored), released once\n * ownership is assumed.\n */\n purgeLock: PurgeLock | null;\n /**\n * Requests a restart of the change stream (at a transaction boundary),\n * which is used to start pending backfills once ownership is assumed.\n */\n requestRestart: (reason: string) => void;\n};\n\ntype ReplicationOwner = {\n owner: string | null;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\nexport type TuningOptions = {\n backPressureLimitHeapProportion: number;\n statementTimeoutMs: number;\n changeLogBatchSize: number;\n drainTimeoutMs?: number | undefined;\n};\n\n/**\n * A single `changeLog` row, accumulated in {@link PendingTransaction.batch}\n * and written via `json_to_recordset()` (see `#flushChangeLog()`).\n */\ntype ChangeLogRow = {\n watermark: string;\n precommit: string | null;\n pos: number;\n change: string;\n};\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n * `replicaVersion`, which is the version at which initial sync\n * was performed, and any attempts to catchup from a different\n * point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n * a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n * subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n readonly id = 'storer';\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #taskID: string;\n readonly #discoveryAddress: string;\n readonly #discoveryProtocol: string;\n readonly #makeConnectionPool: PostgresDBProvider;\n\n // Isolated connection pools for steady-state, potentially concurrent\n // queries, in order to avoid a postgres.js connection-swapping bug:\n // https://github.com/porsager/postgres/issues/1204\n readonly #db: PostgresDB;\n readonly #inserter: PostgresDB;\n readonly #purger: PostgresDB;\n\n readonly #replicaVersion: string;\n readonly #onCommitted: (c: Commit) => void;\n readonly #onFatal: (err: Error) => void;\n\n /**\n * Set while waiting to assume ownership of the change log at its head\n * (see {@link assumeOwnershipAtHead}).\n */\n #pendingOwnership:\n | (OwnershipTakeover & {\n /** The last known head of the change log, which only moves forward. */\n head: string;\n /** Whether a backfill transaction was skipped. */\n skippedBackfill: boolean;\n start: number;\n })\n | null = null;\n readonly #queue = new Queue<QueueEntry>();\n readonly #backPressureThresholdBytes: number;\n readonly #statementTimeoutMs: number;\n readonly #changeLogBatchSize: number;\n readonly #drainTimeoutMs: number;\n readonly #progressMonitor: ProgressMonitor;\n\n #approximateQueuedBytes = 0;\n #running = false;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n dbProvider: PostgresDBProvider,\n replicaVersion: string,\n onCommitted: (c: Commit | UpstreamStatusMessage) => void,\n onFatal: (err: Error) => void,\n {\n backPressureLimitHeapProportion,\n statementTimeoutMs,\n changeLogBatchSize,\n drainTimeoutMs = 30_000,\n }: TuningOptions,\n ) {\n this.#lc = lc.withContext('component', 'change-log');\n this.#shard = shard;\n this.#taskID = taskID;\n this.#discoveryAddress = discoveryAddress;\n this.#discoveryProtocol = discoveryProtocol;\n this.#makeConnectionPool = dbProvider;\n this.#db = dbProvider('change-stream-init', 3);\n this.#inserter = dbProvider('change-log-inserter', 1);\n this.#purger = dbProvider('change-log-purger', 1);\n this.#replicaVersion = replicaVersion;\n this.#onCommitted = onCommitted;\n this.#onFatal = onFatal;\n this.#statementTimeoutMs = statementTimeoutMs;\n this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);\n this.#drainTimeoutMs = drainTimeoutMs;\n this.#progressMonitor = new ProgressMonitor(\n lc,\n statementTimeoutMs,\n onFatal,\n );\n\n const heapStats = getHeapStatistics();\n this.#backPressureThresholdBytes =\n (heapStats.heap_size_limit - heapStats.used_heap_size) *\n backPressureLimitHeapProportion;\n\n this.#lc.info?.(\n `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n `to absorb upstream spikes`,\n {heapStats},\n );\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n /**\n * Bounds a one-off db call (i.e. not part of the main storer loop or the\n * background catchup read, which are tracked by the ProgressMonitor\n * instead) with a plain timeout. This covers calls like\n * {@link assumeOwnership} and {@link getStartStreamInitializationParameters}\n * that are made by the caller *before* {@link run()} -- and thus before the\n * ProgressMonitor's polling starts -- as well as ones made well after,\n * where a continuously-polling watchdog would be overkill for a single\n * bounded db round trip.\n *\n * Does not cancel `promise` on timeout (there is no cancellation mechanism\n * for a postgres.js query); it just stops waiting for it, and swallows a\n * later rejection so it doesn't surface as an unhandled rejection.\n */\n async #withTimeout<T>(name: string, promise: Promise<T>): Promise<T> {\n const result = await orTimeout(promise, this.#statementTimeoutMs);\n if (result === 'timed-out') {\n void promise.catch(() => {});\n throw new AbortError(\n `${name} did not complete within ${this.#statementTimeoutMs}ms`,\n );\n }\n return result;\n }\n\n #ownerAddress() {\n const ownerAddress = this.#discoveryAddress;\n const ownerProtocol = this.#discoveryProtocol;\n // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n return ownerProtocol === 'ws'\n ? ownerAddress\n : `${ownerProtocol}://${ownerAddress}`;\n }\n\n /**\n * Re-initializes a change log that is no longer maintained from the\n * replica's state, taking ownership of it in the process (which is then\n * assumed again at its new head, see {@link assumeOwnershipAtHead}).\n * See {@link reinitializeChangeLog}.\n */\n reinitializeFromReplica(\n params: Omit<ChangeLogReinitialization, 'owner' | 'ownerAddress'>,\n ): Promise<'reinitialized' | 'maintained'> {\n return reinitializeChangeLog(\n this.#lc,\n this.#db,\n this.#shard,\n {...params, owner: this.#taskID, ownerAddress: this.#ownerAddress()},\n // The purge lock (if any) is released before the change log is\n // truncated, as it locks a row of it.\n this.#pendingOwnership?.purgeLock ?? undefined,\n );\n }\n\n /**\n * Prepares to assume ownership of the change log at its head, while it may\n * still be owned and maintained by another change-streamer. This allows the\n * previous owner to continue to serve subscribers while this change-source's\n * upstream decodes the WAL to catch up to the head of the change log, which\n * may be many minutes worth of transactions for large DBs.\n *\n * Once the change stream is started from the head of the change log:\n *\n * * Transactions that the change log already contains (i.e. stored by the\n * previous owner) are not stored again, but are otherwise processed\n * normally (e.g. for upstream acks, since they have been stored).\n * * Ownership is assumed in the transaction that stores the first\n * transaction past the head of the change log. The previous owner's\n * (subsequent) transactions fail its ownership check, which makes this\n * transaction the one immediately after the previous owner's last one.\n * * Ownership is also assumed upon a {@link status} at or past the head of\n * the change log (outside of a transaction), which covers an idle\n * upstream: all transactions up to the status position have been\n * received, and any past the head would have been stored.\n *\n * Subscribers are caught up from the change log as usual, as the stream\n * (which is forwarded to them) starts from its head.\n *\n * Backfill transactions are synthesized by each change-streamer's change\n * source, and thus never correspond to those stored by the previous owner.\n * Until ownership is assumed, they are not stored, and no backfills are\n * requested from the change log (see\n * {@link getStartStreamInitializationParameters}). Once ownership is\n * assumed, a restart of the change stream is requested if the change log has\n * pending backfills or if a backfill transaction was skipped.\n */\n assumeOwnershipAtHead(takeover: OwnershipTakeover) {\n this.#pendingOwnership = {\n ...takeover,\n head: '',\n skippedBackfill: false,\n start: performance.now(),\n };\n }\n\n /**\n * Whether ownership of the change log is pending, i.e. after\n * {@link assumeOwnershipAtHead} and before ownership is assumed.\n */\n get ownershipPending(): boolean {\n return this.#pendingOwnership !== null;\n }\n\n /**\n * Relays a status message received outside of a transaction, which is\n * used to assume ownership of the change log on an idle upstream. A no-op\n * unless ownership is pending.\n */\n status(watermark: string) {\n if (this.#pendingOwnership) {\n this.#queue.enqueue(['status', watermark]);\n }\n }\n\n /**\n * Assumes ownership if the change log contains nothing past `watermark`.\n * Otherwise, updates the known head of the change log.\n *\n * @param ifBefore When set, ownership is only assumed if the head of the\n * change log is before (rather than at or before) the `watermark`,\n * i.e. when the watermark is that of a transaction to be stored.\n * @returns The state of the change log at which ownership is assumed (in\n * `tx`), or `null` if ownership was not assumed.\n */\n async #tryAssumeOwnership(\n tx: TransactionPool,\n watermark: string,\n ifBefore: boolean,\n ): Promise<AssumedOwnership | null> {\n const pending = must(this.#pendingOwnership);\n const [state] = await tx.processReadTask(\n sql => sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')} FOR UPDATE`,\n );\n const head = state.lastWatermark;\n pending.head = head > pending.head ? head : pending.head;\n if (ifBefore ? head >= watermark : head > watermark) {\n return null;\n }\n const owner = this.#taskID;\n const ownerAddress = this.#ownerAddress();\n void tx.process(sql => [\n sql`UPDATE ${this.#cdc('replicationState')} SET ${sql({owner, ownerAddress})}`,\n ]);\n // Read with the replicationState row locked, i.e. consistent with the head.\n const [{backfilling}] = await tx.processReadTask(\n sql => sql<{backfilling: boolean}[]>`\n SELECT EXISTS (SELECT 1 FROM ${this.#cdc('backfilling')}) AS \"backfilling\"`,\n );\n this.#lc.info?.(\n `assuming ownership at ${ownerAddress} after change log head ${head}`,\n );\n return {head, backfilling};\n }\n\n #ownershipAssumed({head, backfilling}: AssumedOwnership) {\n const {purgeLock, requestRestart, skippedBackfill, start} = must(\n this.#pendingOwnership,\n );\n this.#pendingOwnership = null;\n this.#lc.info?.(\n `assumed ownership of the change log at ${head} after ` +\n `${(performance.now() - start).toFixed(2)} ms`,\n );\n // Once ownership has been assumed, the initial purge lock can be released,\n // as a change-streamer that was attempting to purge records will\n // correspondingly abort on the ownership check.\n void purgeLock?.release();\n if (backfilling || skippedBackfill) {\n requestRestart(`resuming backfills after assuming ownership`);\n }\n }\n\n /**\n * Assumes ownership of the change log immediately, regardless of its head.\n * This is used for in tests.\n *\n * A production change-streamer instead takes over the change log at its head\n * (see {@link assumeOwnershipAtHead}).\n */\n async assumeOwnership() {\n const db = this.#db;\n const owner = this.#taskID;\n const addressWithProtocol = this.#ownerAddress();\n this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n const start = performance.now();\n await this.#withTimeout(\n 'assume-ownership',\n db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`,\n );\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n );\n }\n\n /**\n * The whole of what a stream connection starts from: where to resume, what to\n * ask the change source to backfill, and the raw cookie set those requests\n * were derived from.\n *\n * All four statements run in one REPEATABLE READ transaction, so they see one\n * snapshot. That is invariant 15 — the resume watermark and the cookies must\n * come from the same store, read at the same position — and it is why the\n * cookies are read here rather than by a second call: a cookie set paired\n * with a watermark it was not folded to loses every backfill that completed\n * in the interval, and those rows are not replayable from the slot.\n *\n * `cookies` is not `backfillRequests` in another shape. The request list is\n * driven off `backfilling`, so it drops the metadata of every table with no\n * in-flight backfill — correct for starting a stream, lossy as the cookie\n * snapshot that the SQLite change log continues folding onto.\n */\n async getStartStreamInitializationParameters(): Promise<{\n lastWatermark: string;\n backfillRequests: BackfillRequest[];\n cookies: CookieSet;\n }> {\n const [[{lastWatermark}], result, tableMetadata, backfilling] =\n await this.#withTimeout(\n 'get-stream-params',\n runTx(\n this.#db,\n sql => [\n sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n // Formats a BackfillRequest using json_object_agg() to construct the\n // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n // to make it optional and possibly `null`. The change log does not\n // track backfill progress, so the columns have no `progress`\n // (i.e. they are backfilled from scratch).\n sql`\n SELECT\n json_build_object(\n 'schema', b.\"schema\",\n 'name', b.\"table\",\n 'metadata', t.\"metadata\"\n ) as \"table\",\n json_object_agg(b.\"column\", json_build_object('id', b.\"backfill\"))\n as \"columns\"\n FROM ${this.#cdc('backfilling')} as b\n LEFT JOIN ${this.#cdc('tableMetadata')} as t\n ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n `,\n\n // Ordered by primary key, so that this set and the SQLite change log's\n // can be compared row for row. `COLLATE \"C\"` because that comparison is\n // against a store whose TEXT columns sort by byte: a linguistic\n // collation orders `foo-bar` and `foobar` differently than SQLite's\n // BINARY does, which would read as a divergence rather than as the\n // identical set it is.\n sql<TableMetadataCookie[]>`\n SELECT \"schema\", \"table\", \"metadata\" FROM ${this.#cdc('tableMetadata')}\n ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\"`,\n\n sql<BackfillCookie[]>`\n SELECT \"schema\", \"table\", \"column\", \"backfill\" FROM ${this.#cdc('backfilling')}\n ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\", \"column\" COLLATE \"C\"`,\n ],\n {mode: Mode.READONLY},\n ),\n );\n\n return {\n lastWatermark,\n // Backfills are not requested before ownership is assumed.\n // See assumeOwnershipAtHead().\n backfillRequests: this.#pendingOwnership\n ? []\n : v.parse(result, backfillRequestsSchema),\n cookies: {\n tableMetadata: [...tableMetadata],\n backfilling: [...backfilling],\n },\n };\n }\n\n async getMinWatermarkForCatchup(): Promise<string | null> {\n const [{minWatermark}] = await this.#withTimeout(\n 'get-min-watermark',\n this.#db<{minWatermark: string | null}[]> /*sql*/ `\n SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`,\n );\n return minWatermark;\n }\n\n /**\n * Returns the retained Postgres catchup bounds.\n * `minWatermark` is the oldest transaction, or `null` for a new replica.\n * `lastWatermark` is the durable head.\n */\n async getCatchupBounds(): Promise<{\n minWatermark: string | null;\n lastWatermark: string;\n }> {\n const [bounds] = await this.#withTimeout(\n 'get-catchup-bounds',\n this.#db<{minWatermark: string | null; lastWatermark: string}[]> /*sql*/ `\n SELECT\n (SELECT min(watermark) FROM ${this.#cdc('changeLog')}) as \"minWatermark\",\n (SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}) as \"lastWatermark\"`,\n );\n return bounds;\n }\n\n /**\n * Lists at most `limit` committed transaction watermarks in\n * `(afterWatermark, throughWatermark]`, in ascending order.\n */\n async listCommitWatermarks(\n afterWatermark: string,\n throughWatermark: string,\n limit: number,\n ): Promise<string[]> {\n const rows = await this.#db<{watermark: string}[]> /*sql*/ `\n SELECT watermark FROM ${this.#cdc('changeLog')}\n WHERE precommit IS NOT NULL\n AND watermark > ${afterWatermark}\n AND watermark <= ${throughWatermark}\n ORDER BY watermark\n LIMIT ${limit}`;\n return rows.map(({watermark}) => watermark);\n }\n\n /**\n * Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.\n * This query matches the subscriber catchup query.\n *\n * Both queries fail when an escaped NUL reaches `change->'tag'`.\n * Postgres cannot convert that value to text.\n */\n readCatchupRange(\n afterWatermark: string,\n throughWatermark: string,\n batchRows = 2000,\n ): AsyncIterable<ChangeLogEntry[]> {\n assert(\n Number.isSafeInteger(batchRows) && batchRows > 0,\n 'Postgres change log batch size must be a positive safe integer',\n );\n return this.#db<ChangeLogEntry[]> /*sql*/ `\n SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n WHERE watermark > ${afterWatermark}\n AND watermark <= ${throughWatermark}\n ORDER BY watermark, pos`.cursor(batchRows) as AsyncIterable<\n ChangeLogEntry[]\n >;\n }\n\n // Note: These are the only db `await`s that are not tracked by the\n // ProgressMonitor, as the delete can legitimately block and/or take\n // an arbitrary amount of time. Luckily, this is a background call and\n // thus cannot cause the storer loop to hang.\n purgeRecordsBefore(watermark: string): Promise<number> {\n return runTx(this.#purger, async sql => {\n // This NOWAIT pre-check is an optimization to abort the transaction\n // (and release associated resources) early.\n await sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR UPDATE NOWAIT\n `;\n // If the row is purge-locked by an incoming replication-manager, it\n // will assume ownership of the change-log before releasing the lock.\n // This DELETE blocks until the lock is released, allowing the change\n // in ownership to be reliably detected (and the transaction aborted)\n // in the subsequent check.\n const [{deleted}] = await sql<{deleted: bigint}[]>`\n -- The backup watermark can be ahead of the durable changeLog if the\n -- storer is behind but the backup replica has consumed forwarded\n -- changes. Preserve the latest durable changeLog transaction as the\n -- catchup boundary instead of assuming the backup watermark exists.\n -- The storer inserts each changeLog transaction atomically, so any\n -- durable row for a watermark implies the full transaction is durable.\n WITH keep AS (\n SELECT max(watermark) AS watermark\n FROM ${this.#cdc('changeLog')}\n ), purged AS (\n DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n AND watermark < (SELECT watermark FROM keep)\n RETURNING watermark, pos\n ) SELECT COUNT(*) as deleted FROM purged;`;\n\n const [{owner}] = await sql<ReplicationOwner[]>`\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR SHARE`;\n if (owner !== this.#taskID) {\n throw new AbortError(\n `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n );\n }\n return Number(deleted);\n });\n }\n\n /**\n * @returns The JSON stringified stream message to be sent downstream.\n */\n store(\n watermark: string,\n data: ChangeStreamData,\n serialized: SerializedChangeStreamData = serializeChangeStreamDataWithChange(\n data,\n ),\n ) {\n // Eagerly stringify the JSON payload to:\n // - avoid redundant stringification when fanning out to subscribers\n // - efficiently estimate the amount of memory the payload consumes\n const {json, change: storedChange} = serialized;\n this.#approximateQueuedBytes += json.length;\n\n const change = data[1];\n this.#queue.enqueue([\n 'change',\n watermark,\n json,\n storedChange,\n isDataChange(change) ? null : change, // drop DataChanges to save memory\n ]);\n\n return json;\n }\n\n abort() {\n this.#queue.enqueue(['abort']);\n }\n\n catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n }\n\n #readyForMore: Resolver<void> | null = null;\n\n readyForMore(): Promise<void> | undefined {\n if (!this.#running) {\n return undefined;\n }\n if (\n this.#readyForMore === null &&\n this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n ) {\n this.#lc.warn?.(\n `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n `\\n` +\n `To inspect changeLog backlog in your change DB:\\n` +\n ` SELECT\\n` +\n ` (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n ` change->>'tag' AS operation,\\n` +\n ` COUNT(*) AS count\\n` +\n ` FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n ` GROUP BY 1, 2\\n` +\n ` ORDER BY 3 DESC\\n` +\n ` LIMIT 20;`,\n );\n this.#readyForMore = resolver();\n }\n return this.#readyForMore?.promise;\n }\n\n #maybeReleaseBackPressure() {\n if (this.#readyForMore !== null) {\n // Wait for at least 20% of the threshold to free up.\n if (\n this.#approximateQueuedBytes <\n this.#backPressureThresholdBytes * 0.8\n ) {\n this.#lc.info?.(\n `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n );\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n }\n }\n\n /**\n * Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.\n *\n * Uses `json_to_recordset()` so the batch is a single JSON parameter: the\n * statement text stays constant regardless of batch size, avoiding the\n * unbounded prepared-statement variants (and Postgres memory growth) that a\n * multi-row INSERT would produce. See rocicorp/mono#3511.\n *\n * Returns (and updates) {@link PendingTransaction.lastFlush}; a no-op when the\n * batch is empty.\n */\n #flushChangeLog(tx: PendingTransaction): Promise<unknown> | undefined {\n const {batch} = tx;\n if (batch.length === 0) {\n return tx.lastFlush;\n }\n tx.batch = [];\n tx.lastFlush = tx.pool.process(sql => [\n sql`\n INSERT INTO ${this.#cdc('changeLog')} (\"watermark\", \"pos\", \"change\", \"precommit\")\n SELECT \"watermark\", \"pos\", \"change\"::json, \"precommit\"\n FROM json_to_recordset(${batch}) AS x(\n \"watermark\" TEXT,\n \"pos\" INT8,\n \"change\" TEXT,\n \"precommit\" TEXT\n )`,\n ]);\n return tx.lastFlush;\n }\n\n #stopped = promiseVoid;\n\n /**\n * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n * Once {@link run()} completes, it can be called again.\n */\n async run() {\n assert(!this.#running, `storer is already running`);\n\n const {promise: stopped, resolve: signalStopped} = resolver();\n this.#running = true;\n this.#stopped = stopped;\n\n this.#lc.info?.('starting storer');\n this.#progressMonitor.start(); // Note: This is stopped in stop()\n let err: unknown;\n try {\n await this.#processQueue();\n } catch (e) {\n err = e; // used in finally\n throw e;\n } finally {\n // Release any pending backpressure so the upstream can proceed\n if (this.#readyForMore !== null) {\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n this.#cancelQueueEntries(\n this.#queue.drain().filter(entry => entry !== undefined),\n err,\n );\n this.#running = false;\n signalStopped();\n this.#lc.info?.('storer stopped');\n }\n }\n\n #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n if (queue.length === 0) {\n return;\n }\n this.#lc.info?.(\n `canceling ${queue.length} entries from the changeLog queue`,\n );\n const err = e instanceof Error ? e : new AbortError('server shutting down');\n for (const entry of queue) {\n if (entry === 'stop') {\n continue;\n }\n const type = entry[0];\n switch (type) {\n case 'subscriber': {\n // Disconnect subscribers waiting to be caught up so that they can\n // reconnect and try again.\n const {subscriber} = entry[1];\n this.#lc.info?.(`disconnecting ${subscriber.id}`);\n subscriber.fail(err);\n break;\n }\n }\n }\n }\n\n async #processQueue() {\n let tx: PendingTransaction | SkippedTransaction | null = null;\n let msg: QueueEntry | false;\n\n // Track the progress of each (previous and next) queue entry before it is\n // processed in the loop.\n let lastTaskDone: TaskDoneFn | undefined;\n const nextMessage = async () => {\n lastTaskDone?.();\n lastTaskDone = undefined;\n const entry = await this.#queue.dequeue();\n if (entry !== 'stop') {\n lastTaskDone = this.#progressMonitor.trackTask({\n task: 'queue-entry',\n // Note: TaskKeys are logged, so only include entry[0] (the type) to\n // avoid logging application data.\n type: entry[0],\n });\n }\n return entry;\n };\n\n const catchupQueue: SubscriberAndMode[] = [];\n try {\n while ((msg = await nextMessage()) !== 'stop') {\n const [msgType] = msg;\n switch (msgType) {\n case 'ready': {\n const signalReady = msg[1];\n signalReady();\n continue;\n }\n case 'subscriber': {\n const subscriber = msg[1];\n if (tx) {\n catchupQueue.push(subscriber); // Wait for the current tx to complete.\n } else {\n await this.#startCatchup([subscriber]); // Catch up immediately.\n }\n continue;\n }\n case 'status': {\n const pending = this.#pendingOwnership;\n if (pending && !tx && msg[1] >= pending.head) {\n const pool = new TransactionPool(this.#lc, {\n mode: Mode.READ_COMMITTED,\n });\n pool.run(this.#inserter);\n const assumed = await this.#tryAssumeOwnership(\n pool,\n msg[1],\n false,\n );\n if (assumed === null) {\n pool.abort();\n await pool.done();\n } else {\n pool.setDone();\n await pool.done();\n this.#ownershipAssumed(assumed);\n }\n }\n continue;\n }\n case 'abort': {\n if (tx && !tx.skipped) {\n tx.pool.abort();\n await tx.pool.done();\n }\n tx = null;\n continue;\n }\n }\n // msgType === 'change'\n this.#approximateQueuedBytes -= msg[2].length;\n tx = await this.#processChange(tx, msg);\n // Released once the change has been processed (e.g. flushed), so that\n // the upstream is held back by the progress of the change log.\n this.#maybeReleaseBackPressure();\n if (tx === null) {\n // Before beginning the next transaction, open a READONLY snapshot to\n // concurrently catchup subscribers that were waiting for the\n // transaction to end.\n await this.#startCatchup(catchupQueue.splice(0));\n }\n }\n // Subscribers waiting for a transaction that was interrupted (i.e.\n // aborted before stopping) are disconnected so that they can reconnect,\n // like those that are still in the queue.\n this.#cancelQueueEntries(\n catchupQueue.splice(0).map(sub => ['subscriber', sub]),\n undefined,\n );\n } catch (e) {\n catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n }\n\n /**\n * Processes a change from the queue, i.e. stores it in the change log as\n * part of the current transaction (or skips it, see\n * {@link assumeOwnershipAtHead}).\n *\n * @returns The transaction that is open after the change, or `null` if the\n * change ended (or was outside of) a transaction.\n */\n async #processChange(\n tx: PendingTransaction | SkippedTransaction | null,\n msg: Extract<QueueEntry, ['change', ...unknown[]]>,\n ): Promise<PendingTransaction | SkippedTransaction | null> {\n const [_, watermark, json, storedChange, change] = msg;\n const tag = change?.tag;\n\n if (tag === 'begin') {\n assert(!tx, 'received BEGIN in the middle of a transaction');\n const pending = this.#pendingOwnership;\n if (pending) {\n // Backfill transactions are neither stored nor used to assume\n // ownership. See assumeOwnershipAtHead().\n if (change.skipAck) {\n pending.skippedBackfill = true;\n return {skipped: true, ack: false};\n }\n if (watermark <= pending.head) {\n return {skipped: true, ack: true};\n }\n }\n const pool = new TransactionPool(\n this.#lc.withContext('watermark', watermark),\n {mode: Mode.READ_COMMITTED},\n );\n pool.run(this.#inserter);\n\n const {promise, resolve, reject} = resolver<ReplicationOwner>();\n void promise.catch(() => {}); // handle rejections before the await\n\n const assumed = pending\n ? await this.#tryAssumeOwnership(pool, watermark, true)\n : undefined;\n if (assumed === null) {\n // Stored by the previous owner in the meantime.\n pool.abort();\n await pool.done();\n return {skipped: true, ack: true};\n }\n if (pending) {\n // Ownership was assumed in this transaction.\n resolve({owner: this.#taskID});\n } else {\n // Acquire a lock on the replicationState row to detect and/or prevent\n // a concurrent ownership change.\n void pool.process(tx => {\n tx<ReplicationOwner[]> /*sql*/ `\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n ([result]) => resolve(result),\n reject,\n );\n return [];\n });\n }\n\n tx = {\n pool,\n preCommitWatermark: watermark,\n pos: 0,\n startingReplicationState: promise,\n ack: !change.skipAck,\n batch: [],\n lastFlush: undefined,\n ownershipAssumed: assumed,\n };\n } else if (tx?.skipped) {\n if (tag === 'commit' && tx.ack) {\n this.#onCommitted(['commit', change, {watermark}]);\n }\n return tag === 'commit' || tag === 'rollback' ? null : tx;\n } else {\n assert(tx, () => `received change outside of transaction: ${json}`);\n tx.pos++;\n }\n\n const entry: ChangeLogRow = {\n watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n pos: tx.pos,\n // For backwards compatibility, only the change message is stored\n // in the cdc changeLog. It was serialized separately with the\n // downstream envelope, so no full-message scan is needed here.\n change: storedChange,\n };\n\n if (change !== null && isSchemaChange(change)) {\n // Schema changes carry backfill / table-metadata statements that\n // must be applied in stream order relative to the changeLog rows.\n // Flush any buffered rows first, then write this row together with\n // its metadata statements as a single unit (preserving the previous\n // per-change ordering for schema changes).\n await this.#flushChangeLog(tx);\n tx.lastFlush = tx.pool.process(sql => [\n sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n ...this.#trackBackfillMetadata(sql, change),\n ]);\n } else {\n // Accumulate plain changeLog rows (begin, data changes, commit) and\n // write them as a single multi-row INSERT. Collapsing the per-change\n // single-row INSERTs into batches is the dominant cost reduction for\n // large transactions, where the previous one-statement-per-change\n // path dominated the upstream replication lag.\n tx.batch.push(entry);\n if (tx.batch.length >= this.#changeLogBatchSize) {\n // Bound pipeline depth (and thus memory) by awaiting the previous\n // flush before issuing the next. This is the batched analog of the\n // previous per-100-statement backpressure await, and likewise\n // guards against memory blowup on very large transactions.\n const prevFlush = tx.lastFlush;\n void this.#flushChangeLog(tx);\n await prevFlush;\n }\n }\n if (tag === 'commit') {\n // Flush any remaining buffered changeLog rows (including this commit\n // row) before updating the replication state, so the state update is\n // ordered after all changeLog inserts for this transaction.\n void this.#flushChangeLog(tx);\n\n const {owner} = await tx.startingReplicationState;\n if (owner !== this.#taskID) {\n // Ownership change reflected in the replicationState read in 'begin'.\n tx.pool.fail(\n new AbortError(`changeLog ownership has been assumed by ${owner}`),\n );\n } else {\n // Update the replication state.\n const lastWatermark = watermark;\n void tx.pool.process(tx => [\n tx`\n UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n ]);\n tx.pool.setDone();\n }\n\n await tx.pool.done();\n\n const {ownershipAssumed, ack} = tx;\n tx = null;\n if (ownershipAssumed) {\n this.#ownershipAssumed(ownershipAssumed);\n }\n // ACK the LSN to the upstream Postgres.\n if (ack) {\n this.#onCommitted(['commit', change, {watermark}]);\n }\n } else if (tag === 'rollback') {\n // Aborted transactions are not stored in the changeLog.\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n }\n return tx;\n }\n\n async #startCatchup(subs: SubscriberAndMode[]) {\n const numCatchups = subs.length;\n if (numCatchups === 0) {\n return;\n }\n\n const lc = this.#lc.withContext('pool', 'catchup');\n const {init, cleanup, snapshotID} = sharedSnapshot();\n const reader = new TransactionPool(lc, {\n mode: Mode.READONLY,\n init,\n cleanup,\n initialWorkers: subs.length,\n });\n\n // A dynamic connection pool is created for catchup, sized to the number\n // of subscribers being caught up.\n const catchupConns = this.#makeConnectionPool(\n 'subscriber-catchup',\n numCatchups,\n );\n reader.run(catchupConns);\n\n let lastWatermark: string | undefined;\n const catchupSnapshotted = this.#progressMonitor.trackTask({\n task: 'capture-catchup-snapshot',\n subscribers: subs.map(({subscriber: s}) => s.id),\n });\n try {\n // Ensure that the transaction has started (and is thus holding a snapshot\n // of the database) before continuing on to commit more changes. This is\n // done by performing a single read on the db, which determines the\n // snapshot for the REPEATABLE_READ transaction.\n [{lastWatermark}] = await reader.processReadTask(\n sql => sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}\n `,\n );\n lc.info?.(\n `snapshotted db at ${lastWatermark} (${await snapshotID}) ` +\n `to catchup ${numCatchups} subscriber(s)`,\n );\n } catch (e) {\n subs.map(({subscriber}) => subscriber.fail(e));\n // Nothing will run the catchup tasks that normally finalize the pool\n // (below), so tear it down here. Otherwise its workers (each holding an\n // open READ ONLY transaction) and the connection pool are leaked.\n reader.abort();\n void catchupConns.end().catch(() => {});\n throw e;\n } finally {\n catchupSnapshotted();\n }\n\n // Run the actual catchup queries in the background. Errors are handled in\n // #catchup() by disconnecting the associated subscriber.\n void Promise.allSettled(\n subs.map(sub =>\n this.#catchup(\n lc.withContext('subscriber', sub.subscriber.id),\n sub,\n lastWatermark,\n reader,\n ),\n ),\n ).finally(() => {\n reader.setDone();\n void catchupConns.end().catch(() => {});\n });\n }\n\n async #catchup(\n lc: LogContext,\n {subscriber: sub, mode}: SubscriberAndMode,\n lastWatermark: string,\n reader: TransactionPool,\n ) {\n let entriesReceived = 0;\n let catchupProgressed = this.#progressMonitor.trackTask({\n task: 'catchup',\n subscriber: sub.id,\n entriesReceived,\n });\n try {\n lc.info?.(`starting catchup`);\n await reader.processReadTask(async tx => {\n lc.info?.(`catching up`);\n const start = Date.now();\n\n // When starting from initial-sync, there won't be a change with a watermark\n // equal to the replica version. This is the empty changeLog scenario.\n let watermarkFound = sub.watermark === this.#replicaVersion;\n let count = 0;\n let lastBatchConsumed: Promise<unknown> | undefined;\n\n for await (const entries of tx<ChangeLogEntry[]> /*sql*/ `\n SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n WHERE watermark >= ${sub.watermark}\n AND watermark <= ${lastWatermark}\n ORDER BY watermark, pos`.cursor(2000)) {\n catchupProgressed();\n entriesReceived += entries.length;\n\n try {\n // Wait for the last batch of entries to be consumed by the\n // subscriber before sending down the current batch. This pipelining\n // allows one batch of changes to be received from the change-db\n // while the previous batch of changes are sent to the subscriber,\n // resulting in flow control that caps the number of changes\n // referenced in memory to 2 * batch-size.\n const start = performance.now();\n await lastBatchConsumed;\n const elapsed = performance.now() - start;\n if (lastBatchConsumed) {\n lc[elapsed > 100 ? 'info' : 'debug']?.(\n `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n );\n }\n\n for (const entry of entries) {\n if (entry.watermark === sub.watermark) {\n // This should be the first entry.\n // Catchup starts from *after* the watermark.\n watermarkFound = true;\n } else if (watermarkFound) {\n lastBatchConsumed = sub.catchup(\n reconstructWatermarkedChange(entry),\n );\n count++;\n } else if (mode === 'backup') {\n throw new AutoResetSignal(\n `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n );\n } else {\n lc.warn?.(\n `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n );\n sub.close(\n ErrorType.WatermarkTooOld,\n `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n );\n return;\n }\n }\n } finally {\n // Track the db's read of the next chunk of changes.\n catchupProgressed = this.#progressMonitor.trackTask({\n task: 'catchup',\n subscriber: sub.id,\n entriesReceived,\n });\n }\n }\n catchupProgressed();\n\n if (watermarkFound) {\n await lastBatchConsumed;\n lc.info?.(\n `caught up ${sub.id} with ${count} changes (${\n Date.now() - start\n } ms)`,\n );\n } else {\n // The subscriber is ahead of the latest durable changeLog entry\n // (lastWatermark). This can legitimately happen: changes are\n // forwarded to subscribers (the backup replica and view-syncers)\n // concurrently with — and can outrun — the durable store, so a\n // replica may briefly lead the change DB after the storer falls\n // behind or the change-streamer restarts. No catchup is possible or\n // needed; once the change DB catches back up, forwarding resumes and\n // the subscriber dedups any watermarks it already has. Unlike the\n // AutoResetSignal / WatermarkTooOld cases above, this is not a gap in\n // replication history, so the subscriber is simply marked caught up.\n lc.warn?.(\n `subscriber ${sub.id} at watermark ${sub.watermark} is ahead of ` +\n `the latest durable watermark ${lastWatermark}; waiting for the ` +\n `change DB to catch up`,\n );\n }\n // Start draining messages buffered during catchup. The returned promise\n // is intentionally not awaited here: while the drain is in progress,\n // new sends keep appending to the subscriber backlog and inherit its\n // byte-based backpressure.\n void sub.setCaughtUp();\n });\n } catch (err) {\n lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n if (err instanceof AutoResetSignal) {\n await markResetRequired(this.#db, this.#shard);\n this.#onFatal(err);\n }\n sub.fail(err);\n } finally {\n catchupProgressed();\n }\n }\n\n /**\n * Returns the db statements necessary to track backfill and table metadata\n * presented in the `change`, if any.\n *\n * This is the Postgres interpreter of {@link cookieOps}, which is also\n * interpreted against the SQLite change log by `ChangeLogCookieWriter`. The\n * two stores have to agree on every transition forever, and the failure mode\n * if they drift is a backfill that is silently never re-requested — so the\n * decision of what a schema change *means* lives in one place and only its\n * transport lives here.\n */\n #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n return cookieOps(change).flatMap(op => this.#cookieStmts(sql, op));\n }\n\n #cookieStmts(sql: PostgresTransaction, op: CookieOp): PendingQuery<Row[]>[] {\n switch (op.op) {\n case 'upsert-metadata':\n return [this.#upsertTableMetadataStmt(sql, op.table, op.metadata)];\n\n case 'upsert-backfill':\n return [\n this.#upsertColumnBackfillStmt(sql, op.table, op.column, op.backfill),\n ];\n\n case 'rename-table': {\n const {old} = op;\n const row = {schema: op.new.schema, table: op.new.name};\n return [\n sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n ];\n }\n\n case 'drop-table': {\n const {schema, name} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('tableMetadata')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n ];\n }\n\n case 'rename-column': {\n const {schema, name: table} = op.table;\n return [\n sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${op.new}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${op.old}`,\n ];\n }\n\n case 'drop-column': {\n const {schema, name} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${op.column}`,\n ];\n }\n\n case 'complete-backfill': {\n const {schema, name: table} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql([...op.columns])}`,\n ];\n }\n\n default:\n unreachable(op);\n }\n }\n\n #upsertTableMetadataStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n metadata: TableMetadata,\n ) {\n const row: TableMetadataRow = {schema, table, metadata};\n return sql`\n INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\")\n DO UPDATE SET ${sql(row)};\n `;\n }\n\n #upsertColumnBackfillStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n column: string,\n backfill: BackfillID,\n ) {\n const row: BackfillingColumn = {schema, table, column, backfill};\n return sql`\n INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\", \"column\")\n DO UPDATE SET ${sql(row)};\n `;\n }\n\n /**\n * Waits until all currently queued entries have been processed.\n * This is only used in tests.\n */\n async allProcessed() {\n if (this.#running) {\n const {promise, resolve} = resolver();\n this.#queue.enqueue(['ready', resolve]);\n await promise;\n }\n }\n\n /**\n * Stops the storer and waits up to a drain timeout for entries to drain,\n * throwing an exception if it doesn't drain in time, in order to abort\n * the server when PG (or the connection to it) appears to be wedged.\n */\n async stop() {\n if (this.#running) {\n this.#progressMonitor.stop();\n this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n this.abort(); // for cleanliness, abort any open transactions\n this.#queue.enqueue('stop');\n }\n if (\n (await orTimeout(this.#stopped, this.#drainTimeoutMs)) === 'timed-out'\n ) {\n throw new AbortError(\n `changeLog did not drain within ${this.#drainTimeoutMs}ms`,\n );\n }\n }\n}\n\nexport class PurgeLock {\n readonly #lc: LogContext;\n readonly #tx: TransactionPool;\n readonly replicaVersion: string;\n readonly minWatermark: string;\n /**\n * The head of the change log (i.e. its `lastWatermark`) when the lock was\n * acquired. The change log is resumed from its head, which only advances,\n * so a stream cannot (correctly) be started from before it.\n */\n readonly headWatermark: string;\n\n constructor(\n lc: LogContext,\n tx: TransactionPool,\n replicaVersion: string,\n watermark: string,\n headWatermark: string,\n ) {\n this.#lc = lc;\n this.#tx = tx;\n this.replicaVersion = replicaVersion;\n this.minWatermark = watermark;\n this.headWatermark = headWatermark;\n }\n\n #released = false;\n\n async release() {\n if (this.#released) {\n return;\n }\n this.#released = true;\n this.#tx.setDone();\n await this.#tx\n .done()\n .catch(e => this.#lc.warn?.(`error from purge-lock release`, e));\n this.#lc.info?.(`released purge lock on ${this.minWatermark}`);\n }\n}\n\nexport class PurgeLocker {\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #db: PostgresDB;\n\n constructor(lc: LogContext, shard: ShardID, db: PostgresDB) {\n this.#lc = lc.withContext('component', 'purge-locker');\n this.#shard = shard;\n this.#db = db;\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n /**\n * Locks the oldest row of the change log, preventing it from being purged.\n *\n * @param slotWatermark The position of the replication slot from which the\n * change log is to be resumed (as a watermark). If specified and the\n * head of the change log is behind it, the change log may be missing\n * transactions that the slot will not stream, in which case nothing is\n * locked and `'behind-slot'` is returned. Conversely, a head at or past\n * the slot's position means that the stream (which resumes from the\n * head) will not skip anything.\n * @returns `null` if the change log is empty.\n */\n acquire(): Promise<PurgeLock | null>;\n acquire(\n slotWatermark?: string | undefined,\n ): Promise<PurgeLock | null | 'behind-slot'>;\n async acquire(\n slotWatermark?: string,\n ): Promise<PurgeLock | null | 'behind-slot'> {\n const tx = new TransactionPool(this.#lc, {mode: Mode.READ_COMMITTED}).run(\n this.#db,\n );\n const [state] = await tx.processReadTask(\n sql => sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n );\n // An uninitialized change log (i.e. without a head) is empty, and thus\n // behind any replication slot.\n const lastWatermark = state?.lastWatermark;\n if (\n slotWatermark !== undefined &&\n (lastWatermark === undefined || lastWatermark < slotWatermark)\n ) {\n this.#lc.info?.(\n `changeLog@${lastWatermark ?? '(uninitialized)'} is behind the replication slot@${slotWatermark}`,\n );\n tx.setDone();\n await tx.done();\n return 'behind-slot';\n }\n if (lastWatermark === undefined) {\n this.#lc.info?.(`changeLog is uninitialized. No rows to purge-lock.`);\n tx.setDone();\n await tx.done();\n return null;\n }\n const row = await tx.processReadTask(\n sql => sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR SHARE \n `,\n );\n if (row.length === 0) {\n this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);\n tx.setDone();\n await tx.done();\n return null;\n }\n const [{watermark}] = row;\n const [{replicaVersion}] = await tx.processReadTask(\n sql => sql<{replicaVersion: string}[]>`\n SELECT \"replicaVersion\" FROM ${this.#cdc('replicationConfig')}\n `,\n );\n this.#lc.info?.(\n `locked watermark ${watermark} from being purged from replica@${replicaVersion}`,\n );\n return new PurgeLock(\n this.#lc,\n tx,\n replicaVersion,\n watermark,\n lastWatermark,\n );\n }\n}\n\n// A TaskKey, keyed by identity, representing an db task monitored by the\n// ProgressMonitor. The contents are stringified in error messages if the\n// task fails to make progress for more than a failure threshold interval.\ntype TaskKey = JSONObject;\n\ntype TaskDoneFn = () => void;\n\n/**\n * Periodic monitor for detecting the pathological condition in which a db\n * query hangs forever because of a connection problem, such as a half-closed\n * socket.\n *\n * Use this for db calls that recur or accumulate progress within a single\n * logical operation on the main storer loop (e.g. the queue-processing loop,\n * or a multi-batch catchup read), where \"no progress for N ms\" is the\n * meaningful signal. For a one-off db call -- including ones made before\n * {@link Storer.run} starts this monitor's polling -- use a plain timeout\n * (see `Storer.#withTimeout`) instead.\n *\n * The failure handler should stop the server. The ProgressMonitor will\n * assume this and stop itself after calling it.\n *\n * Internal to the Storer, but exported for testing.\n */\nexport class ProgressMonitor {\n readonly #lc: LogContext;\n readonly #failureThresholdMs;\n readonly #onFailure: (err: Error) => void;\n readonly #taskUpdates = new Map<TaskKey, Date>();\n\n #timer: NodeJS.Timeout | undefined;\n\n constructor(\n lc: LogContext,\n failureThresholdMs: number,\n onFailure: (err: Error) => void,\n ) {\n this.#lc = lc.withContext('component', 'storer-progress-monitor');\n this.#failureThresholdMs = failureThresholdMs;\n this.#onFailure = onFailure;\n }\n\n start() {\n clearInterval(this.#timer);\n this.#timer = setInterval(\n () => this.checkTaskProgress(new Date()),\n this.#failureThresholdMs,\n ).unref();\n }\n\n // Called by an internal periodic timer, but exported for testing.\n checkTaskProgress(now: Date) {\n for (const [task, lastUpdate] of this.#taskUpdates.entries()) {\n if (now.getTime() - lastUpdate.getTime() >= this.#failureThresholdMs) {\n this.#lc.error?.(\n `Last task update was more than ${this.#failureThresholdMs}ms ago`,\n {task, lastUpdate},\n );\n this.#onFailure(\n new Error(\n `Task failed to progress for over ${this.#failureThresholdMs}ms: ${BigIntJSON.stringify(task)}`,\n ),\n );\n this.stop();\n return;\n }\n }\n }\n\n /**\n * Starts a task to be tracked.\n *\n * @return a callback to invoke when the task is done.\n */\n trackTask(task: TaskKey, now = new Date()): TaskDoneFn {\n this.#taskUpdates.set(task, now);\n return () => {\n this.#taskUpdates.delete(task);\n };\n }\n\n stop() {\n clearInterval(this.#timer);\n this.#taskUpdates.clear();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA8IA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkD5D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CAKA;CACA;CACA;CAEA;CACA;CACA;;;;;CAMA,oBAQW;CACX,SAAkB,IAAI,MAAkB;CACxC;CACA;CACA;CACA;CACA;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,YACA,gBACA,aACA,SACA,EACE,iCACA,oBACA,oBACA,iBAAiB,OAEnB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKC,SAAS;EACd,KAAKC,UAAU;EACf,KAAKC,oBAAoB;EACzB,KAAKC,qBAAqB;EAC1B,KAAKC,sBAAsB;EAC3B,KAAKC,MAAM,WAAW,sBAAsB,CAAC;EAC7C,KAAKC,YAAY,WAAW,uBAAuB,CAAC;EACpD,KAAKC,UAAU,WAAW,qBAAqB,CAAC;EAChD,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;EACpB,KAAKC,WAAW;EAChB,KAAKG,sBAAsB;EAC3B,KAAKC,sBAAsB,KAAK,IAAI,GAAG,kBAAkB;EACzD,KAAKC,kBAAkB;EACvB,KAAKC,mBAAmB,IAAI,gBAC1B,IACA,oBACA,OACF;EAEA,MAAM,YAAY,kBAAkB;EACpC,KAAKJ,+BACF,UAAU,kBAAkB,UAAU,kBACvC;EAEF,KAAKb,IAAI,OACP,gBAAgB,KAAKa,8BAA8B,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAE/E,EAAC,UAAS,CACZ;CACF;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKP,IAAI,GAAG,UAAU,KAAKL,MAAM,EAAE,GAAG,OAAO;CACtD;;;;;;;;;;;;;;;CAgBA,MAAMiB,aAAgB,MAAc,SAAiC;EACnE,MAAM,SAAS,MAAM,UAAU,SAAS,KAAKJ,mBAAmB;EAChE,IAAI,WAAW,aAAa;GAC1B,QAAa,YAAY,CAAC,CAAC;GAC3B,MAAM,IAAI,WACR,GAAG,KAAK,2BAA2B,KAAKA,oBAAoB,GAC9D;EACF;EACA,OAAO;CACT;CAEA,gBAAgB;EACd,MAAM,eAAe,KAAKX;EAC1B,MAAM,gBAAgB,KAAKC;EAE3B,OAAO,kBAAkB,OACrB,eACA,GAAG,cAAc,KAAK;CAC5B;;;;;;;CAQA,wBACE,QACyC;EACzC,OAAO,sBACL,KAAKJ,KACL,KAAKM,KACL,KAAKL,QACL;GAAC,GAAG;GAAQ,OAAO,KAAKC;GAAS,cAAc,KAAKiB,cAAc;EAAC,GAGnE,KAAKC,mBAAmB,aAAa,KAAA,CACvC;CACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkCA,sBAAsB,UAA6B;EACjD,KAAKA,oBAAoB;GACvB,GAAG;GACH,MAAM;GACN,iBAAiB;GACjB,OAAO,YAAY,IAAI;EACzB;CACF;;;;;CAMA,IAAI,mBAA4B;EAC9B,OAAO,KAAKA,sBAAsB;CACpC;;;;;;CAOA,OAAO,WAAmB;EACxB,IAAI,KAAKA,mBACP,KAAKR,OAAO,QAAQ,CAAC,UAAU,SAAS,CAAC;CAE7C;;;;;;;;;;;CAYA,MAAMS,oBACJ,IACA,WACA,UACkC;EAClC,MAAM,UAAU,KAAK,KAAKD,iBAAiB;EAC3C,MAAM,CAAC,SAAS,MAAM,GAAG,iBACvB,QAAO,GAA8B;sCACL,KAAKE,KAAK,kBAAkB,EAAE,YAChE;EACA,MAAM,OAAO,MAAM;EACnB,QAAQ,OAAO,OAAO,QAAQ,OAAO,OAAO,QAAQ;EACpD,IAAI,WAAW,QAAQ,YAAY,OAAO,WACxC,OAAO;EAET,MAAM,QAAQ,KAAKpB;EACnB,MAAM,eAAe,KAAKiB,cAAc;EACxC,GAAQ,SAAQ,QAAO,CACrB,GAAG,UAAU,KAAKG,KAAK,kBAAkB,EAAE,OAAO,IAAI;GAAC;GAAO;EAAY,CAAC,GAC7E,CAAC;EAED,MAAM,CAAC,EAAC,iBAAgB,MAAM,GAAG,iBAC/B,QAAO,GAA6B;uCACH,KAAKA,KAAK,aAAa,EAAE,mBAC5D;EACA,KAAKtB,IAAI,OACP,yBAAyB,aAAa,yBAAyB,MACjE;EACA,OAAO;GAAC;GAAM;EAAW;CAC3B;CAEA,kBAAkB,EAAC,MAAM,eAAgC;EACvD,MAAM,EAAC,WAAW,gBAAgB,iBAAiB,UAAS,KAC1D,KAAKoB,iBACP;EACA,KAAKA,oBAAoB;EACzB,KAAKpB,IAAI,OACP,0CAA0C,KAAK,UACzC,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC,EAAE,IAC9C;EAIA,WAAgB,QAAQ;EACxB,IAAI,eAAe,iBACjB,eAAe,6CAA6C;CAEhE;;;;;;;;CASA,MAAM,kBAAkB;EACtB,MAAM,KAAK,KAAKM;EAChB,MAAM,QAAQ,KAAKJ;EACnB,MAAM,sBAAsB,KAAKiB,cAAc;EAC/C,KAAKnB,IAAI,OAAO,yBAAyB,qBAAqB;EAC9D,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAKkB,aACT,oBACA,EAAE,UAAU,KAAKI,KAAK,kBAAkB,EAAE,OAAO,GAAG;GAAC;GAAO,cAAc;EAAmB,CAAC,GAChG;EACA,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,KAAKtB,IAAI,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,KAC1D;CACF;;;;;;;;;;;;;;;;;;CAmBA,MAAM,yCAIH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,QAAQ,eAAe,eAC/C,MAAM,KAAKkB,aACT,qBACA,MACE,KAAKZ,MACL,QAAO;GACL,GAA8B;sCACJ,KAAKgB,KAAK,kBAAkB;GAOtD,GAAG;;;;;;;;;iBASE,KAAKA,KAAK,aAAa,EAAE;sBACpB,KAAKA,KAAK,eAAe,EAAE;;;;GAWrC,GAA0B;oDACc,KAAKA,KAAK,eAAe,EAAE;;GAGnE,GAAqB;8DAC6B,KAAKA,KAAK,aAAa,EAAE;;EAE7E,GACA,EAAC,MAAM,SAAa,CACtB,CACF;EAEF,OAAO;GACL;GAGA,kBAAkB,KAAKF,oBACnB,CAAC,IACD,MAAQ,QAAQ,sBAAsB;GAC1C,SAAS;IACP,eAAe,CAAC,GAAG,aAAa;IAChC,aAAa,CAAC,GAAG,WAAW;GAC9B;EACF;CACF;CAEA,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,KAAKF,aAClC,qBACA,KAAKZ,GAA6C;qDACH,KAAKgB,KAAK,WAAW,GACtE;EACA,OAAO;CACT;;;;;;CAOA,MAAM,mBAGH;EACD,MAAM,CAAC,UAAU,MAAM,KAAKJ,aAC1B,sBACA,KAAKZ,GAAoE;;sCAEzC,KAAKgB,KAAK,WAAW,EAAE;uCACtB,KAAKA,KAAK,kBAAkB,EAAE,qBACjE;EACA,OAAO;CACT;;;;;CAMA,MAAM,qBACJ,gBACA,kBACA,OACmB;EAQnB,QAAO,MAPY,KAAKhB,GAAmC;8BACjC,KAAKgB,KAAK,WAAW,EAAE;;2BAE1B,eAAe;4BACd,iBAAiB;;eAE9B,SACC,KAAK,EAAC,gBAAe,SAAS;CAC5C;;;;;;;;CASA,iBACE,gBACA,kBACA,YAAY,KACqB;EACjC,OACE,OAAO,cAAc,SAAS,KAAK,YAAY,GAC/C,gEACF;EACA,OAAO,KAAKhB,GAA8B;kEACoB,KAAKgB,KAAK,WAAW,EAAE;2BAC9D,eAAe;4BACd,iBAAiB;gCACb,OAAO,SAAS;CAG9C;CAMA,mBAAmB,WAAoC;EACrD,OAAO,MAAM,KAAKd,SAAS,OAAM,QAAO;GAGtC,MAAM,GAA0B;kCACJ,KAAKc,KAAK,WAAW,EAAE;;;;GASnD,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;;;;;;;;iBASvC,KAAKA,KAAK,WAAW,EAAE;;wBAEhB,KAAKA,KAAK,WAAW,EAAE,qBAAqB,UAAU;;;;GAKxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;8BACvB,KAAKA,KAAK,kBAAkB,EAAE;GACtD,IAAI,UAAU,KAAKpB,SACjB,MAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,OAClF;GAEF,OAAO,OAAO,OAAO;EACvB,CAAC;CACH;;;;CAKA,MACE,WACA,MACA,aAAyC,oCACvC,IACF,GACA;EAIA,MAAM,EAAC,MAAM,QAAQ,iBAAgB;EACrC,KAAKqB,2BAA2B,KAAK;EAErC,MAAM,SAAS,KAAK;EACpB,KAAKX,OAAO,QAAQ;GAClB;GACA;GACA;GACA;GACA,aAAa,MAAM,IAAI,OAAO;EAChC,CAAC;EAED,OAAO;CACT;CAEA,QAAQ;EACN,KAAKA,OAAO,QAAQ,CAAC,OAAO,CAAC;CAC/B;CAEA,QAAQ,YAAwB,MAAsB;EACpD,KAAKA,OAAO,QAAQ,CAAC,cAAc;GAAC;GAAY;EAAI,CAAC,CAAC;CACxD;CAEA,gBAAuC;CAEvC,eAA0C;EACxC,IAAI,CAAC,KAAKY,UACR;EAEF,IACE,KAAKC,kBAAkB,QACvB,KAAKF,0BAA0B,KAAKV,6BACpC;GACA,KAAKb,IAAI,OACP,+BAA+B,KAAKY,OAAO,KAAK,EAAE,qBAAqB,KAAKW,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,2SAW9H;GACA,KAAKE,gBAAgB,SAAS;EAChC;EACA,OAAO,KAAKA,eAAe;CAC7B;CAEA,4BAA4B;EAC1B,IAAI,KAAKA,kBAAkB;OAGvB,KAAKF,0BACL,KAAKV,8BAA8B,IACnC;IACA,KAAKb,IAAI,OACP,gCAAgC,KAAKY,OAAO,KAAK,EAAE,qBAAqB,KAAKW,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,KAC/H;IACA,KAAKE,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;;CAEJ;;;;;;;;;;;;CAaA,gBAAgB,IAAsD;EACpE,MAAM,EAAC,UAAS;EAChB,IAAI,MAAM,WAAW,GACnB,OAAO,GAAG;EAEZ,GAAG,QAAQ,CAAC;EACZ,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG;sBACa,KAAKH,KAAK,WAAW,EAAE;;mCAEV,MAAM;;;;;YAMrC,CAAC;EACD,OAAO,GAAG;CACZ;CAEA,WAAW;;;;;CAMX,MAAM,MAAM;EACV,OAAO,CAAC,KAAKE,UAAU,2BAA2B;EAElD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,SAAS;EAC5D,KAAKA,WAAW;EAChB,KAAKE,WAAW;EAEhB,KAAK1B,IAAI,OAAO,iBAAiB;EACjC,KAAKiB,iBAAiB,MAAM;EAC5B,IAAI;EACJ,IAAI;GACF,MAAM,KAAKU,cAAc;EAC3B,SAAS,GAAG;GACV,MAAM;GACN,MAAM;EACR,UAAU;GAER,IAAI,KAAKF,kBAAkB,MAAM;IAC/B,KAAKA,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;GACA,KAAKG,oBACH,KAAKhB,OAAO,MAAM,EAAE,QAAO,UAAS,UAAU,KAAA,CAAS,GACvD,GACF;GACA,KAAKY,WAAW;GAChB,cAAc;GACd,KAAKxB,IAAI,OAAO,gBAAgB;EAClC;CACF;CAEA,oBAAoB,OAAqB,GAAY;EACnD,IAAI,MAAM,WAAW,GACnB;EAEF,KAAKA,IAAI,OACP,aAAa,MAAM,OAAO,kCAC5B;EACA,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,sBAAsB;EAC1E,KAAK,MAAM,SAAS,OAAO;GACzB,IAAI,UAAU,QACZ;GAGF,QADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;KAC3B,KAAKA,IAAI,OAAO,iBAAiB,WAAW,IAAI;KAChD,WAAW,KAAK,GAAG;KACnB;IACF;GACF;EACF;CACF;CAEA,MAAM2B,gBAAgB;EACpB,IAAI,KAAqD;EACzD,IAAI;EAIJ,IAAI;EACJ,MAAM,cAAc,YAAY;GAC9B,eAAe;GACf,eAAe,KAAA;GACf,MAAM,QAAQ,MAAM,KAAKf,OAAO,QAAQ;GACxC,IAAI,UAAU,QACZ,eAAe,KAAKK,iBAAiB,UAAU;IAC7C,MAAM;IAGN,MAAM,MAAM;GACd,CAAC;GAEH,OAAO;EACT;EAEA,MAAM,eAAoC,CAAC;EAC3C,IAAI;GACF,QAAQ,MAAM,MAAM,YAAY,OAAO,QAAQ;IAC7C,MAAM,CAAC,WAAW;IAClB,QAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;MACxB,YAAY;MACZ;KACF;KACA,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;MACvB,IAAI,IACF,aAAa,KAAK,UAAU;WAE5B,MAAM,KAAKY,cAAc,CAAC,UAAU,CAAC;MAEvC;KACF;KACA,KAAK,UAAU;MACb,MAAM,UAAU,KAAKT;MACrB,IAAI,WAAW,CAAC,MAAM,IAAI,MAAM,QAAQ,MAAM;OAC5C,MAAM,OAAO,IAAI,gBAAgB,KAAKpB,KAAK,EACzC,MAAM,eACR,CAAC;OACD,KAAK,IAAI,KAAKO,SAAS;OACvB,MAAM,UAAU,MAAM,KAAKc,oBACzB,MACA,IAAI,IACJ,KACF;OACA,IAAI,YAAY,MAAM;QACpB,KAAK,MAAM;QACX,MAAM,KAAK,KAAK;OAClB,OAAO;QACL,KAAK,QAAQ;QACb,MAAM,KAAK,KAAK;QAChB,KAAKS,kBAAkB,OAAO;OAChC;MACF;MACA;KACF;KACA,KAAK;MACH,IAAI,MAAM,CAAC,GAAG,SAAS;OACrB,GAAG,KAAK,MAAM;OACd,MAAM,GAAG,KAAK,KAAK;MACrB;MACA,KAAK;MACL;IAEJ;IAEA,KAAKP,2BAA2B,IAAI,GAAG;IACvC,KAAK,MAAM,KAAKQ,eAAe,IAAI,GAAG;IAGtC,KAAKC,0BAA0B;IAC/B,IAAI,OAAO,MAIT,MAAM,KAAKH,cAAc,aAAa,OAAO,CAAC,CAAC;GAEnD;GAIA,KAAKD,oBACH,aAAa,OAAO,CAAC,EAAE,KAAI,QAAO,CAAC,cAAc,GAAG,CAAC,GACrD,KAAA,CACF;EACF,SAAS,GAAG;GACV,aAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GACzD,MAAM;EACR;CACF;;;;;;;;;CAUA,MAAMG,eACJ,IACA,KACyD;EACzD,MAAM,CAAC,GAAG,WAAW,MAAM,cAAc,UAAU;EACnD,MAAM,MAAM,QAAQ;EAEpB,IAAI,QAAQ,SAAS;GACnB,OAAO,CAAC,IAAI,+CAA+C;GAC3D,MAAM,UAAU,KAAKX;GACrB,IAAI,SAAS;IAGX,IAAI,OAAO,SAAS;KAClB,QAAQ,kBAAkB;KAC1B,OAAO;MAAC,SAAS;MAAM,KAAK;KAAK;IACnC;IACA,IAAI,aAAa,QAAQ,MACvB,OAAO;KAAC,SAAS;KAAM,KAAK;IAAI;GAEpC;GACA,MAAM,OAAO,IAAI,gBACf,KAAKpB,IAAI,YAAY,aAAa,SAAS,GAC3C,EAAC,MAAM,eAAmB,CAC5B;GACA,KAAK,IAAI,KAAKO,SAAS;GAEvB,MAAM,EAAC,SAAS,SAAS,WAAU,SAA2B;GAC9D,QAAa,YAAY,CAAC,CAAC;GAE3B,MAAM,UAAU,UACZ,MAAM,KAAKc,oBAAoB,MAAM,WAAW,IAAI,IACpD,KAAA;GACJ,IAAI,YAAY,MAAM;IAEpB,KAAK,MAAM;IACX,MAAM,KAAK,KAAK;IAChB,OAAO;KAAC,SAAS;KAAM,KAAK;IAAI;GAClC;GACA,IAAI,SAEF,QAAQ,EAAC,OAAO,KAAKnB,QAAO,CAAC;QAI7B,KAAU,SAAQ,OAAM;IACtB,EAA+B;kCACP,KAAKoB,KAAK,kBAAkB,EAAE,aAAa,MAChE,CAAC,YAAY,QAAQ,MAAM,GAC5B,MACF;IACA,OAAO,CAAC;GACV,CAAC;GAGH,KAAK;IACH;IACA,oBAAoB;IACpB,KAAK;IACL,0BAA0B;IAC1B,KAAK,CAAC,OAAO;IACb,OAAO,CAAC;IACR,WAAW,KAAA;IACX,kBAAkB;GACpB;EACF,OAAO,IAAI,IAAI,SAAS;GACtB,IAAI,QAAQ,YAAY,GAAG,KACzB,KAAKZ,aAAa;IAAC;IAAU;IAAQ,EAAC,UAAS;GAAC,CAAC;GAEnD,OAAO,QAAQ,YAAY,QAAQ,aAAa,OAAO;EACzD,OAAO;GACL,OAAO,UAAU,2CAA2C,MAAM;GAClE,GAAG;EACL;EAEA,MAAM,QAAsB;GAC1B,WAAW,QAAQ,WAAW,YAAY,GAAG;GAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;GACtD,KAAK,GAAG;GAIR,QAAQ;EACV;EAEA,IAAI,WAAW,QAAQ,eAAe,MAAM,GAAG;GAM7C,MAAM,KAAKuB,gBAAgB,EAAE;GAC7B,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG,eAAe,KAAKX,KAAK,WAAW,EAAE,GAAG,IAAI,KAAK,KACrD,GAAG,KAAKY,uBAAuB,KAAK,MAAM,CAC5C,CAAC;EACH,OAAO;GAML,GAAG,MAAM,KAAK,KAAK;GACnB,IAAI,GAAG,MAAM,UAAU,KAAKnB,qBAAqB;IAK/C,MAAM,YAAY,GAAG;IACrB,KAAUkB,gBAAgB,EAAE;IAC5B,MAAM;GACR;EACF;EACA,IAAI,QAAQ,UAAU;GAIpB,KAAUA,gBAAgB,EAAE;GAE5B,MAAM,EAAC,UAAS,MAAM,GAAG;GACzB,IAAI,UAAU,KAAK/B,SAEjB,GAAG,KAAK,KACN,IAAI,WAAW,2CAA2C,OAAO,CACnE;QACK;IAEL,MAAM,gBAAgB;IACtB,GAAQ,KAAK,SAAQ,OAAM,CACzB,EAAE;iBACK,KAAKoB,KAAK,kBAAkB,EAAE,OAAO,GAAG,EAAC,cAAa,CAAC,GAChE,CAAC;IACD,GAAG,KAAK,QAAQ;GAClB;GAEA,MAAM,GAAG,KAAK,KAAK;GAEnB,MAAM,EAAC,kBAAkB,QAAO;GAChC,KAAK;GACL,IAAI,kBACF,KAAKQ,kBAAkB,gBAAgB;GAGzC,IAAI,KACF,KAAKpB,aAAa;IAAC;IAAU;IAAQ,EAAC,UAAS;GAAC,CAAC;EAErD,OAAO,IAAI,QAAQ,YAAY;GAE7B,GAAG,KAAK,MAAM;GACd,MAAM,GAAG,KAAK,KAAK;GACnB,KAAK;EACP;EACA,OAAO;CACT;CAEA,MAAMmB,cAAc,MAA2B;EAC7C,MAAM,cAAc,KAAK;EACzB,IAAI,gBAAgB,GAClB;EAGF,MAAM,KAAK,KAAK7B,IAAI,YAAY,QAAQ,SAAS;EACjD,MAAM,EAAC,MAAM,SAAS,eAAc,eAAe;EACnD,MAAM,SAAS,IAAI,gBAAgB,IAAI;GACrC,MAAM;GACN;GACA;GACA,gBAAgB,KAAK;EACvB,CAAC;EAID,MAAM,eAAe,KAAKK,oBACxB,sBACA,WACF;EACA,OAAO,IAAI,YAAY;EAEvB,IAAI;EACJ,MAAM,qBAAqB,KAAKY,iBAAiB,UAAU;GACzD,MAAM;GACN,aAAa,KAAK,KAAK,EAAC,YAAY,QAAO,EAAE,EAAE;EACjD,CAAC;EACD,IAAI;GAKF,CAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAA8B;sCACP,KAAKK,KAAK,kBAAkB,EAAE;OAE9D;GACA,GAAG,OACD,qBAAqB,cAAc,IAAI,MAAM,WAAW,eACxC,YAAY,eAC9B;EACF,SAAS,GAAG;GACV,KAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GAI7C,OAAO,MAAM;GACb,aAAkB,IAAI,EAAE,YAAY,CAAC,CAAC;GACtC,MAAM;EACR,UAAU;GACR,mBAAmB;EACrB;EAIA,QAAa,WACX,KAAK,KAAI,QACP,KAAKa,SACH,GAAG,YAAY,cAAc,IAAI,WAAW,EAAE,GAC9C,KACA,eACA,MACF,CACF,CACF,EAAE,cAAc;GACd,OAAO,QAAQ;GACf,aAAkB,IAAI,EAAE,YAAY,CAAC,CAAC;EACxC,CAAC;CACH;CAEA,MAAMA,SACJ,IACA,EAAC,YAAY,KAAK,QAClB,eACA,QACA;EACA,IAAI,kBAAkB;EACtB,IAAI,oBAAoB,KAAKlB,iBAAiB,UAAU;GACtD,MAAM;GACN,YAAY,IAAI;GAChB;EACF,CAAC;EACD,IAAI;GACF,GAAG,OAAO,kBAAkB;GAC5B,MAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,GAAG,OAAO,aAAa;IACvB,MAAM,QAAQ,KAAK,IAAI;IAIvB,IAAI,iBAAiB,IAAI,cAAc,KAAKR;IAC5C,IAAI,QAAQ;IACZ,IAAI;IAEJ,WAAW,MAAM,WAAW,EAA6B;sEACK,KAAKa,KAAK,WAAW,EAAE;gCAC7D,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,GAAI,GAAG;KACxC,kBAAkB;KAClB,mBAAmB,QAAQ;KAE3B,IAAI;MAOF,MAAM,QAAQ,YAAY,IAAI;MAC9B,MAAM;MACN,MAAM,UAAU,YAAY,IAAI,IAAI;MACpC,IAAI,mBACF,GAAG,UAAU,MAAM,SAAS,WAC1B,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,IAAI,GAAG,0CAChD;MAGF,KAAK,MAAM,SAAS,SAClB,IAAI,MAAM,cAAc,IAAI,WAG1B,iBAAiB;WACZ,IAAI,gBAAgB;OACzB,oBAAoB,IAAI,QACtB,6BAA6B,KAAK,CACpC;OACA;MACF,OAAO,IAAI,SAAS,UAClB,MAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,EACvF;WACK;OACL,GAAG,OACD,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,EAC7F;OACA,IAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,EACjF;OACA;MACF;KAEJ,UAAU;MAER,oBAAoB,KAAKL,iBAAiB,UAAU;OAClD,MAAM;OACN,YAAY,IAAI;OAChB;MACF,CAAC;KACH;IACF;IACA,kBAAkB;IAElB,IAAI,gBAAgB;KAClB,MAAM;KACN,GAAG,OACD,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,IAAI,IAAI,MACd,KACH;IACF,OAWE,GAAG,OACD,cAAc,IAAI,GAAG,gBAAgB,IAAI,UAAU,4CACjB,cAAc,wCAElD;IAMF,IAAS,YAAY;GACvB,CAAC;EACH,SAAS,KAAK;GACZ,GAAG,QAAQ,sCAAsC,IAAI,MAAM,GAAG;GAC9D,IAAI,eAAe,iBAAiB;IAClC,MAAM,kBAAkB,KAAKX,KAAK,KAAKL,MAAM;IAC7C,KAAKU,SAAS,GAAG;GACnB;GACA,IAAI,KAAK,GAAG;EACd,UAAU;GACR,kBAAkB;EACpB;CACF;;;;;;;;;;;;CAaA,uBAAuB,KAA0B,QAAsB;EACrE,OAAO,UAAU,MAAM,EAAE,SAAQ,OAAM,KAAKyB,aAAa,KAAK,EAAE,CAAC;CACnE;CAEA,aAAa,KAA0B,IAAqC;EAC1E,QAAQ,GAAG,IAAX;GACE,KAAK,mBACH,OAAO,CAAC,KAAKC,yBAAyB,KAAK,GAAG,OAAO,GAAG,QAAQ,CAAC;GAEnE,KAAK,mBACH,OAAO,CACL,KAAKC,0BAA0B,KAAK,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CACtE;GAEF,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,GAAG,IAAI;KAAQ,OAAO,GAAG,IAAI;IAAI;IACtD,OAAO,CACL,GAAG,UAAU,KAAKhB,KAAK,eAAe,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,MAC3D;GACF;GAEA,KAAK,cAAc;IACjB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,eAAe,EAAE;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MACnD;GACF;GAEA,KAAK,iBAAiB;IACpB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,kBAAkB,GAAG,IAAI;mCACtC,OAAO,iBAAiB,MAAM,kBAAkB,GAAG,KAC9E;GACF;GAEA,KAAK,eAAe;IAClB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,GAAG,QAC7E;GACF;GAEA,KAAK,qBAAqB;IACxB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAC/F;GACF;GAEA,SACE,YAAY,EAAE;EAClB;CACF;CAEA,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;EAAQ;EACtD,OAAO,GAAG;sBACQ,KAAKA,KAAK,eAAe,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEnC,IAAI,GAAG,EAAE;;CAEjC;CAEA,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;EAAQ;EAC/D,OAAO,GAAG;sBACQ,KAAKA,KAAK,aAAa,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEjC,IAAI,GAAG,EAAE;;CAEjC;;;;;CAMA,MAAM,eAAe;EACnB,IAAI,KAAKE,UAAU;GACjB,MAAM,EAAC,SAAS,YAAW,SAAS;GACpC,KAAKZ,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;GACtC,MAAM;EACR;CACF;;;;;;CAOA,MAAM,OAAO;EACX,IAAI,KAAKY,UAAU;GACjB,KAAKP,iBAAiB,KAAK;GAC3B,KAAKjB,IAAI,OAAO,YAAY,KAAKY,OAAO,KAAK,EAAE,mBAAmB;GAClE,KAAK,MAAM;GACX,KAAKA,OAAO,QAAQ,MAAM;EAC5B;EACA,IACG,MAAM,UAAU,KAAKc,UAAU,KAAKV,eAAe,MAAO,aAE3D,MAAM,IAAI,WACR,kCAAkC,KAAKA,gBAAgB,GACzD;CAEJ;AACF;AAEA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;;;;;;CAMA;CAEA,YACE,IACA,IACA,gBACA,WACA,eACA;EACA,KAAKhB,MAAM;EACX,KAAKuC,MAAM;EACX,KAAK,iBAAiB;EACtB,KAAK,eAAe;EACpB,KAAK,gBAAgB;CACvB;CAEA,YAAY;CAEZ,MAAM,UAAU;EACd,IAAI,KAAKC,WACP;EAEF,KAAKA,YAAY;EACjB,KAAKD,IAAI,QAAQ;EACjB,MAAM,KAAKA,IACR,KAAK,EACL,OAAM,MAAK,KAAKvC,IAAI,OAAO,iCAAiC,CAAC,CAAC;EACjE,KAAKA,IAAI,OAAO,0BAA0B,KAAK,cAAc;CAC/D;AACF;AAEA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB,IAAgB;EAC1D,KAAKA,MAAM,GAAG,YAAY,aAAa,cAAc;EACrD,KAAKC,SAAS;EACd,KAAKK,MAAM;CACb;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKA,IAAI,GAAG,UAAU,KAAKL,MAAM,EAAE,GAAG,OAAO;CACtD;CAkBA,MAAM,QACJ,eAC2C;EAC3C,MAAM,KAAK,IAAI,gBAAgB,KAAKD,KAAK,EAAC,MAAM,eAAmB,CAAC,EAAE,IACpE,KAAKM,GACP;EACA,MAAM,CAAC,SAAS,MAAM,GAAG,iBACvB,QAAO,GAA8B;oCACP,KAAKgB,KAAK,kBAAkB,GAC5D;EAGA,MAAM,gBAAgB,OAAO;EAC7B,IACE,kBAAkB,KAAA,MACjB,kBAAkB,KAAA,KAAa,gBAAgB,gBAChD;GACA,KAAKtB,IAAI,OACP,aAAa,iBAAiB,kBAAkB,kCAAkC,eACpF;GACA,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,IAAI,kBAAkB,KAAA,GAAW;GAC/B,KAAKA,IAAI,OAAO,oDAAoD;GACpE,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,MAAM,MAAM,MAAM,GAAG,iBACnB,QAAO,GAA0B;8BACT,KAAKsB,KAAK,WAAW,EAAE;;;KAIjD;EACA,IAAI,IAAI,WAAW,GAAG;GACpB,KAAKtB,IAAI,OAAO,4CAA4C;GAC5D,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,MAAM,CAAC,EAAC,eAAc;EACtB,MAAM,CAAC,EAAC,oBAAmB,MAAM,GAAG,iBAClC,QAAO,GAA+B;uCACL,KAAKsB,KAAK,mBAAmB,EAAE;OAElE;EACA,KAAKtB,IAAI,OACP,oBAAoB,UAAU,kCAAkC,gBAClE;EACA,OAAO,IAAI,UACT,KAAKA,KACL,IACA,gBACA,WACA,aACF;CACF;AACF;;;;;;;;;;;;;;;;;;AA0BA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA,+BAAwB,IAAI,IAAmB;CAE/C;CAEA,YACE,IACA,oBACA,WACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,yBAAyB;EAChE,KAAKyC,sBAAsB;EAC3B,KAAKC,aAAa;CACpB;CAEA,QAAQ;EACN,cAAc,KAAKE,MAAM;EACzB,KAAKA,SAAS,kBACN,KAAK,kCAAkB,IAAI,KAAK,CAAC,GACvC,KAAKH,mBACP,EAAE,MAAM;CACV;CAGA,kBAAkB,KAAW;EAC3B,KAAK,MAAM,CAAC,MAAM,eAAe,KAAKE,aAAa,QAAQ,GACzD,IAAI,IAAI,QAAQ,IAAI,WAAW,QAAQ,KAAK,KAAKF,qBAAqB;GACpE,KAAKzC,IAAI,QACP,kCAAkC,KAAKyC,oBAAoB,SAC3D;IAAC;IAAM;GAAU,CACnB;GACA,KAAKC,2BACH,IAAI,MACF,oCAAoC,KAAKD,oBAAoB,MAAM,WAAW,UAAU,IAAI,GAC9F,CACF;GACA,KAAK,KAAK;GACV;EACF;CAEJ;;;;;;CAOA,UAAU,MAAe,sBAAM,IAAI,KAAK,GAAe;EACrD,KAAKE,aAAa,IAAI,MAAM,GAAG;EAC/B,aAAa;GACX,KAAKA,aAAa,OAAO,IAAI;EAC/B;CACF;CAEA,OAAO;EACL,cAAc,KAAKC,MAAM;EACzB,KAAKD,aAAa,MAAM;CAC1B;AACF"}
|
|
1
|
+
{"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#makeConnectionPool","#db","#inserter","#purger","#subscriptionState","#replicaVersion","#onCommitted","#onFatal","#queue","#backPressureThresholdBytes","#statementTimeoutMs","#changeLogBatchSize","#drainTimeoutMs","#progressMonitor","#withTimeout","#ownerAddress","#pendingOwnership","#tryAssumeOwnership","#cdc","#approximateQueuedBytes","#running","#readyForMore","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#ownershipAssumed","#processChange","#maybeReleaseBackPressure","#flushChangeLog","#trackBackfillMetadata","#catchup","#cookieStmts","#upsertTableMetadataStmt","#upsertColumnBackfillStmt","#tx","#released","#failureThresholdMs","#onFailure","#taskUpdates","#timer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {getHeapStatistics} from 'node:v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {\n BigIntJSON,\n type JSONObject,\n} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {sharedSnapshot, TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {orTimeout} from '../../types/timeout.ts';\nimport {\n backfillRequestSchema,\n isDataChange,\n isSchemaChange,\n type BackfillID,\n type BackfillRequest,\n type Change,\n type DataChange,\n type Identifier,\n type SchemaChange,\n type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {\n type ChangeStreamData,\n type Commit,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {UpstreamStatusMessage} from '../change-source/protocol/current/status.ts';\nimport {\n cookieOps,\n type BackfillCookie,\n type CookieOp,\n type CookieSet,\n type TableMetadataCookie,\n} from '../replicator/change-log-cookies.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport {\n reconstructWatermarkedChange,\n serializeChangeStreamDataWithChange,\n type ChangeLogEntry,\n type SerializedChangeStreamData,\n} from './change-log-codec.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n AutoResetSignal,\n ensureReplicationConfig,\n markResetRequired,\n reinitializeChangeLog,\n type ReplicaSubscriptionState,\n type BackfillingColumn,\n type ChangeLogReinitialization,\n type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\n/**\n * Factory for creating dynamic connection pools used by the Storer to\n * isolate concurrent transactions into separate pg clients.\n */\nexport type PostgresDBProvider = (\n applicationName: string,\n maxConns: number,\n) => PostgresDB;\n\ntype SubscriberAndMode = {\n subscriber: Subscriber;\n mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n | [\n 'change',\n watermark: string,\n json: string,\n storedChange: string,\n orig: Exclude<Change, DataChange> | null, // null for DataChanges\n ]\n | ['ready', callback: () => void]\n | ['subscriber', SubscriberAndMode]\n | ['status', watermark: string]\n | ['abort']\n | 'stop';\n\n/**\n * A transaction that is not stored because it was already stored by the\n * previous owner of the change log (see {@link Storer.assumeOwnershipAtHead}).\n */\ntype SkippedTransaction = {\n skipped: true;\n ack: boolean;\n};\n\ntype PendingTransaction = {\n skipped?: undefined;\n pool: TransactionPool;\n preCommitWatermark: string;\n pos: number;\n startingReplicationState: Promise<ReplicationOwner>;\n ack: boolean;\n // changeLog rows buffered for the next multi-row INSERT flush.\n batch: ChangeLogRow[];\n // The most recently issued flush (or metadata) process, awaited to bound\n // pipeline depth and to order the commit-time replicationState update.\n lastFlush: Promise<unknown> | undefined;\n // Set if ownership of the change log is assumed in this transaction.\n ownershipAssumed: AssumedOwnership | undefined;\n};\n\n/** The state of the change log at which ownership was assumed. */\ntype AssumedOwnership = {\n /** The last watermark stored by the previous owner. */\n head: string;\n /** Whether the change log has pending backfills. */\n backfilling: boolean;\n};\n\nexport type OwnershipTakeover = {\n /**\n * The lock that prevents the previous owner from purging the change log\n * (from which this change-streamer's replica was restored), released once\n * ownership is assumed.\n */\n purgeLock: PurgeLock | null;\n /**\n * Requests a restart of the change stream (at a transaction boundary),\n * which is used to start pending backfills once ownership is assumed.\n */\n requestRestart: (reason: string) => void;\n /**\n * Whether to throw an AutoResetSignal if the change db requires a reset.\n */\n autoReset: boolean;\n /**\n * Whether the change log was behind the replication slot, in which case it\n * is re-initialized from the replica (see `reinitializeFromReplica()`), and\n * can thus be reset for a replica at any watermark.\n */\n pgChangeLogBehindSlot: boolean;\n};\n\ntype ReplicationOwner = {\n owner: string | null;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\nexport type TuningOptions = {\n backPressureLimitHeapProportion: number;\n statementTimeoutMs: number;\n changeLogBatchSize: number;\n drainTimeoutMs?: number | undefined;\n};\n\n/**\n * A single `changeLog` row, accumulated in {@link PendingTransaction.batch}\n * and written via `json_to_recordset()` (see `#flushChangeLog()`).\n */\ntype ChangeLogRow = {\n watermark: string;\n precommit: string | null;\n pos: number;\n change: string;\n};\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n * `replicaVersion`, which is the version at which initial sync\n * was performed, and any attempts to catchup from a different\n * point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n * a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n * subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n readonly id = 'storer';\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #taskID: string;\n readonly #discoveryAddress: string;\n readonly #discoveryProtocol: string;\n readonly #makeConnectionPool: PostgresDBProvider;\n\n // Isolated connection pools for steady-state, potentially concurrent\n // queries, in order to avoid a postgres.js connection-swapping bug:\n // https://github.com/porsager/postgres/issues/1204\n readonly #db: PostgresDB;\n readonly #inserter: PostgresDB;\n readonly #purger: PostgresDB;\n\n readonly #subscriptionState: ReplicaSubscriptionState;\n readonly #replicaVersion: string;\n readonly #onCommitted: (c: Commit) => void;\n readonly #onFatal: (err: Error) => void;\n\n /**\n * Set while waiting to assume ownership of the change log at its head\n * (see {@link assumeOwnershipAtHead}).\n */\n #pendingOwnership:\n | (OwnershipTakeover & {\n /** The last known head of the change log, which only moves forward. */\n head: string;\n /** Whether a backfill transaction was skipped. */\n skippedBackfill: boolean;\n start: number;\n })\n | null = null;\n readonly #queue = new Queue<QueueEntry>();\n readonly #backPressureThresholdBytes: number;\n readonly #statementTimeoutMs: number;\n readonly #changeLogBatchSize: number;\n readonly #drainTimeoutMs: number;\n readonly #progressMonitor: ProgressMonitor;\n\n #approximateQueuedBytes = 0;\n #running = false;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n dbProvider: PostgresDBProvider,\n subscriptionState: ReplicaSubscriptionState,\n onCommitted: (c: Commit | UpstreamStatusMessage) => void,\n onFatal: (err: Error) => void,\n {\n backPressureLimitHeapProportion,\n statementTimeoutMs,\n changeLogBatchSize,\n drainTimeoutMs = 30_000,\n }: TuningOptions,\n ) {\n this.#lc = lc.withContext('component', 'change-log');\n this.#shard = shard;\n this.#taskID = taskID;\n this.#discoveryAddress = discoveryAddress;\n this.#discoveryProtocol = discoveryProtocol;\n this.#makeConnectionPool = dbProvider;\n this.#db = dbProvider('change-stream-init', 3);\n this.#inserter = dbProvider('change-log-inserter', 1);\n this.#purger = dbProvider('change-log-purger', 1);\n this.#subscriptionState = subscriptionState;\n this.#replicaVersion = subscriptionState.replicaVersion;\n this.#onCommitted = onCommitted;\n this.#onFatal = onFatal;\n this.#statementTimeoutMs = statementTimeoutMs;\n this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);\n this.#drainTimeoutMs = drainTimeoutMs;\n this.#progressMonitor = new ProgressMonitor(\n lc,\n statementTimeoutMs,\n onFatal,\n );\n\n const heapStats = getHeapStatistics();\n this.#backPressureThresholdBytes =\n (heapStats.heap_size_limit - heapStats.used_heap_size) *\n backPressureLimitHeapProportion;\n\n this.#lc.info?.(\n `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n `to absorb upstream spikes`,\n {heapStats},\n );\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n /**\n * Bounds a one-off db call (i.e. not part of the main storer loop or the\n * background catchup read, which are tracked by the ProgressMonitor\n * instead) with a plain timeout. This covers calls like\n * {@link assumeOwnership} and {@link getStartStreamInitializationParameters}\n * that are made by the caller *before* {@link run()} -- and thus before the\n * ProgressMonitor's polling starts -- as well as ones made well after,\n * where a continuously-polling watchdog would be overkill for a single\n * bounded db round trip.\n *\n * Does not cancel `promise` on timeout (there is no cancellation mechanism\n * for a postgres.js query); it just stops waiting for it, and swallows a\n * later rejection so it doesn't surface as an unhandled rejection.\n */\n async #withTimeout<T>(name: string, promise: Promise<T>): Promise<T> {\n const result = await orTimeout(promise, this.#statementTimeoutMs);\n if (result === 'timed-out') {\n void promise.catch(() => {});\n throw new AbortError(\n `${name} did not complete within ${this.#statementTimeoutMs}ms`,\n );\n }\n return result;\n }\n\n #ownerAddress() {\n const ownerAddress = this.#discoveryAddress;\n const ownerProtocol = this.#discoveryProtocol;\n // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n return ownerProtocol === 'ws'\n ? ownerAddress\n : `${ownerProtocol}://${ownerAddress}`;\n }\n\n /**\n * Re-initializes a change log that is no longer maintained from the\n * replica's state, taking ownership of it in the process (which is then\n * assumed again at its new head, see {@link assumeOwnershipAtHead}).\n * See {@link reinitializeChangeLog}.\n */\n reinitializeFromReplica(\n params: Omit<ChangeLogReinitialization, 'owner' | 'ownerAddress'>,\n ): Promise<'reinitialized' | 'maintained'> {\n return reinitializeChangeLog(\n this.#lc,\n this.#db,\n this.#shard,\n {...params, owner: this.#taskID, ownerAddress: this.#ownerAddress()},\n // The purge lock (if any) is released before the change log is\n // truncated, as it locks a row of it.\n this.#pendingOwnership?.purgeLock ?? undefined,\n );\n }\n\n /**\n * Prepares to assume ownership of the change log at its head, while it may\n * still be owned and maintained by another change-streamer. This allows the\n * previous owner to continue to serve subscribers while this change-source's\n * upstream decodes the WAL to catch up to the head of the change log, which\n * may be many minutes worth of transactions for large DBs.\n *\n * Once the change stream is started from the head of the change log:\n *\n * * Transactions that the change log already contains (i.e. stored by the\n * previous owner) are not stored again, but are otherwise processed\n * normally (e.g. for upstream acks, since they have been stored).\n * * Ownership is assumed in the transaction that stores the first\n * transaction past the head of the change log. The previous owner's\n * (subsequent) transactions fail its ownership check, which makes this\n * transaction the one immediately after the previous owner's last one.\n * * Ownership is also assumed upon a {@link status} at or past the head of\n * the change log (outside of a transaction), which covers an idle\n * upstream: all transactions up to the status position have been\n * received, and any past the head would have been stored.\n *\n * Subscribers are caught up from the change log as usual, as the stream\n * (which is forwarded to them) starts from its head.\n *\n * Backfill transactions are synthesized by each change-streamer's change\n * source, and thus never correspond to those stored by the previous owner.\n * Until ownership is assumed, they are not stored, and no backfills are\n * requested from the change log (see\n * {@link getStartStreamInitializationParameters}). Once ownership is\n * assumed, a restart of the change stream is requested if the change log has\n * pending backfills or if a backfill transaction was skipped.\n */\n async assumeOwnershipAtHead(takeover: OwnershipTakeover) {\n // Initialize the cdc tables for the replica, resetting the change log if\n // it was written for a different replica (e.g. after a resync). This is\n // deferred until now (see `checkReplicationConfig()`) because it disrupts\n // the change-streamer that is currently serving the change log.\n await ensureReplicationConfig(\n this.#lc,\n this.#db,\n this.#subscriptionState,\n this.#shard,\n takeover.autoReset,\n takeover.purgeLock ?? undefined,\n undefined,\n takeover.pgChangeLogBehindSlot,\n );\n this.#pendingOwnership = {\n ...takeover,\n head: '',\n skippedBackfill: false,\n start: performance.now(),\n };\n }\n\n /**\n * Whether ownership of the change log is pending, i.e. after\n * {@link assumeOwnershipAtHead} and before ownership is assumed.\n */\n get ownershipPending(): boolean {\n return this.#pendingOwnership !== null;\n }\n\n /**\n * Relays a status message received outside of a transaction, which is\n * used to assume ownership of the change log on an idle upstream. A no-op\n * unless ownership is pending.\n */\n status(watermark: string) {\n if (this.#pendingOwnership) {\n this.#queue.enqueue(['status', watermark]);\n }\n }\n\n /**\n * Assumes ownership if the change log contains nothing past `watermark`.\n * Otherwise, updates the known head of the change log.\n *\n * @param ifBefore When set, ownership is only assumed if the head of the\n * change log is before (rather than at or before) the `watermark`,\n * i.e. when the watermark is that of a transaction to be stored.\n * @returns The state of the change log at which ownership is assumed (in\n * `tx`), or `null` if ownership was not assumed.\n */\n async #tryAssumeOwnership(\n tx: TransactionPool,\n watermark: string,\n ifBefore: boolean,\n ): Promise<AssumedOwnership | null> {\n const pending = must(this.#pendingOwnership);\n const [state] = await tx.processReadTask(\n sql => sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')} FOR UPDATE`,\n );\n const head = state.lastWatermark;\n pending.head = head > pending.head ? head : pending.head;\n if (ifBefore ? head >= watermark : head > watermark) {\n return null;\n }\n const owner = this.#taskID;\n const ownerAddress = this.#ownerAddress();\n void tx.process(sql => [\n sql`UPDATE ${this.#cdc('replicationState')} SET ${sql({owner, ownerAddress})}`,\n ]);\n // Read with the replicationState row locked, i.e. consistent with the head.\n const [{backfilling}] = await tx.processReadTask(\n sql => sql<{backfilling: boolean}[]>`\n SELECT EXISTS (SELECT 1 FROM ${this.#cdc('backfilling')}) AS \"backfilling\"`,\n );\n this.#lc.info?.(\n `assuming ownership at ${ownerAddress} after change log head ${head}`,\n );\n return {head, backfilling};\n }\n\n #ownershipAssumed({head, backfilling}: AssumedOwnership) {\n const {purgeLock, requestRestart, skippedBackfill, start} = must(\n this.#pendingOwnership,\n );\n this.#pendingOwnership = null;\n this.#lc.info?.(\n `assumed ownership of the change log at ${head} after ` +\n `${(performance.now() - start).toFixed(2)} ms`,\n );\n // Once ownership has been assumed, the initial purge lock can be released,\n // as a change-streamer that was attempting to purge records will\n // correspondingly abort on the ownership check.\n void purgeLock?.release();\n if (backfilling || skippedBackfill) {\n requestRestart(`resuming backfills after assuming ownership`);\n }\n }\n\n /**\n * Assumes ownership of the change log immediately, regardless of its head.\n * This is used for in tests.\n *\n * A production change-streamer instead takes over the change log at its head\n * (see {@link assumeOwnershipAtHead}).\n */\n async assumeOwnership() {\n const db = this.#db;\n const owner = this.#taskID;\n const addressWithProtocol = this.#ownerAddress();\n this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n const start = performance.now();\n await this.#withTimeout(\n 'assume-ownership',\n db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`,\n );\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n );\n }\n\n /**\n * The whole of what a stream connection starts from: where to resume, what to\n * ask the change source to backfill, and the raw cookie set those requests\n * were derived from.\n *\n * All four statements run in one REPEATABLE READ transaction, so they see one\n * snapshot. That is invariant 15 — the resume watermark and the cookies must\n * come from the same store, read at the same position — and it is why the\n * cookies are read here rather than by a second call: a cookie set paired\n * with a watermark it was not folded to loses every backfill that completed\n * in the interval, and those rows are not replayable from the slot.\n *\n * `cookies` is not `backfillRequests` in another shape. The request list is\n * driven off `backfilling`, so it drops the metadata of every table with no\n * in-flight backfill — correct for starting a stream, lossy as the cookie\n * snapshot that the SQLite change log continues folding onto.\n */\n async getStartStreamInitializationParameters(): Promise<{\n lastWatermark: string;\n backfillRequests: BackfillRequest[];\n cookies: CookieSet;\n }> {\n const [[{lastWatermark}], result, tableMetadata, backfilling] =\n await this.#withTimeout(\n 'get-stream-params',\n runTx(\n this.#db,\n sql => [\n sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n // Formats a BackfillRequest using json_object_agg() to construct the\n // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n // to make it optional and possibly `null`. The change log does not\n // track backfill progress, so the columns have no `progress`\n // (i.e. they are backfilled from scratch).\n sql`\n SELECT\n json_build_object(\n 'schema', b.\"schema\",\n 'name', b.\"table\",\n 'metadata', t.\"metadata\"\n ) as \"table\",\n json_object_agg(b.\"column\", json_build_object('id', b.\"backfill\"))\n as \"columns\"\n FROM ${this.#cdc('backfilling')} as b\n LEFT JOIN ${this.#cdc('tableMetadata')} as t\n ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n `,\n\n // Ordered by primary key, so that this set and the SQLite change log's\n // can be compared row for row. `COLLATE \"C\"` because that comparison is\n // against a store whose TEXT columns sort by byte: a linguistic\n // collation orders `foo-bar` and `foobar` differently than SQLite's\n // BINARY does, which would read as a divergence rather than as the\n // identical set it is.\n sql<TableMetadataCookie[]>`\n SELECT \"schema\", \"table\", \"metadata\" FROM ${this.#cdc('tableMetadata')}\n ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\"`,\n\n sql<BackfillCookie[]>`\n SELECT \"schema\", \"table\", \"column\", \"backfill\" FROM ${this.#cdc('backfilling')}\n ORDER BY \"schema\" COLLATE \"C\", \"table\" COLLATE \"C\", \"column\" COLLATE \"C\"`,\n ],\n {mode: Mode.READONLY},\n ),\n );\n\n return {\n lastWatermark,\n // Backfills are not requested before ownership is assumed.\n // See assumeOwnershipAtHead().\n backfillRequests: this.#pendingOwnership\n ? []\n : v.parse(result, backfillRequestsSchema),\n cookies: {\n tableMetadata: [...tableMetadata],\n backfilling: [...backfilling],\n },\n };\n }\n\n async getMinWatermarkForCatchup(): Promise<string | null> {\n const [{minWatermark}] = await this.#withTimeout(\n 'get-min-watermark',\n this.#db<{minWatermark: string | null}[]> /*sql*/ `\n SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`,\n );\n return minWatermark;\n }\n\n /**\n * Returns the retained Postgres catchup bounds.\n * `minWatermark` is the oldest transaction, or `null` for a new replica.\n * `lastWatermark` is the durable head.\n */\n async getCatchupBounds(): Promise<{\n minWatermark: string | null;\n lastWatermark: string;\n }> {\n const [bounds] = await this.#withTimeout(\n 'get-catchup-bounds',\n this.#db<{minWatermark: string | null; lastWatermark: string}[]> /*sql*/ `\n SELECT\n (SELECT min(watermark) FROM ${this.#cdc('changeLog')}) as \"minWatermark\",\n (SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}) as \"lastWatermark\"`,\n );\n return bounds;\n }\n\n /**\n * Lists at most `limit` committed transaction watermarks in\n * `(afterWatermark, throughWatermark]`, in ascending order.\n */\n async listCommitWatermarks(\n afterWatermark: string,\n throughWatermark: string,\n limit: number,\n ): Promise<string[]> {\n const rows = await this.#db<{watermark: string}[]> /*sql*/ `\n SELECT watermark FROM ${this.#cdc('changeLog')}\n WHERE precommit IS NOT NULL\n AND watermark > ${afterWatermark}\n AND watermark <= ${throughWatermark}\n ORDER BY watermark\n LIMIT ${limit}`;\n return rows.map(({watermark}) => watermark);\n }\n\n /**\n * Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.\n * This query matches the subscriber catchup query.\n *\n * Both queries fail when an escaped NUL reaches `change->'tag'`.\n * Postgres cannot convert that value to text.\n */\n readCatchupRange(\n afterWatermark: string,\n throughWatermark: string,\n batchRows = 2000,\n ): AsyncIterable<ChangeLogEntry[]> {\n assert(\n Number.isSafeInteger(batchRows) && batchRows > 0,\n 'Postgres change log batch size must be a positive safe integer',\n );\n return this.#db<ChangeLogEntry[]> /*sql*/ `\n SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n WHERE watermark > ${afterWatermark}\n AND watermark <= ${throughWatermark}\n ORDER BY watermark, pos`.cursor(batchRows) as AsyncIterable<\n ChangeLogEntry[]\n >;\n }\n\n // Note: These are the only db `await`s that are not tracked by the\n // ProgressMonitor, as the delete can legitimately block and/or take\n // an arbitrary amount of time. Luckily, this is a background call and\n // thus cannot cause the storer loop to hang.\n purgeRecordsBefore(watermark: string): Promise<number> {\n return runTx(this.#purger, async sql => {\n // This NOWAIT pre-check is an optimization to abort the transaction\n // (and release associated resources) early.\n await sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR UPDATE NOWAIT\n `;\n // If the row is purge-locked by an incoming replication-manager, it\n // will assume ownership of the change-log before releasing the lock.\n // This DELETE blocks until the lock is released, allowing the change\n // in ownership to be reliably detected (and the transaction aborted)\n // in the subsequent check.\n const [{deleted}] = await sql<{deleted: bigint}[]>`\n -- The backup watermark can be ahead of the durable changeLog if the\n -- storer is behind but the backup replica has consumed forwarded\n -- changes. Preserve the latest durable changeLog transaction as the\n -- catchup boundary instead of assuming the backup watermark exists.\n -- The storer inserts each changeLog transaction atomically, so any\n -- durable row for a watermark implies the full transaction is durable.\n WITH keep AS (\n SELECT max(watermark) AS watermark\n FROM ${this.#cdc('changeLog')}\n ), purged AS (\n DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n AND watermark < (SELECT watermark FROM keep)\n RETURNING watermark, pos\n ) SELECT COUNT(*) as deleted FROM purged;`;\n\n const [{owner}] = await sql<ReplicationOwner[]>`\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR SHARE`;\n if (owner !== this.#taskID) {\n throw new AbortError(\n `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n );\n }\n return Number(deleted);\n });\n }\n\n /**\n * @returns The JSON stringified stream message to be sent downstream.\n */\n store(\n watermark: string,\n data: ChangeStreamData,\n serialized: SerializedChangeStreamData = serializeChangeStreamDataWithChange(\n data,\n ),\n ) {\n // Eagerly stringify the JSON payload to:\n // - avoid redundant stringification when fanning out to subscribers\n // - efficiently estimate the amount of memory the payload consumes\n const {json, change: storedChange} = serialized;\n this.#approximateQueuedBytes += json.length;\n\n const change = data[1];\n this.#queue.enqueue([\n 'change',\n watermark,\n json,\n storedChange,\n isDataChange(change) ? null : change, // drop DataChanges to save memory\n ]);\n\n return json;\n }\n\n abort() {\n this.#queue.enqueue(['abort']);\n }\n\n catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n }\n\n #readyForMore: Resolver<void> | null = null;\n\n readyForMore(): Promise<void> | undefined {\n if (!this.#running) {\n return undefined;\n }\n if (\n this.#readyForMore === null &&\n this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n ) {\n this.#lc.warn?.(\n `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n `\\n` +\n `To inspect changeLog backlog in your change DB:\\n` +\n ` SELECT\\n` +\n ` (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n ` change->>'tag' AS operation,\\n` +\n ` COUNT(*) AS count\\n` +\n ` FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n ` GROUP BY 1, 2\\n` +\n ` ORDER BY 3 DESC\\n` +\n ` LIMIT 20;`,\n );\n this.#readyForMore = resolver();\n }\n return this.#readyForMore?.promise;\n }\n\n #maybeReleaseBackPressure() {\n if (this.#readyForMore !== null) {\n // Wait for at least 20% of the threshold to free up.\n if (\n this.#approximateQueuedBytes <\n this.#backPressureThresholdBytes * 0.8\n ) {\n this.#lc.info?.(\n `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n );\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n }\n }\n\n /**\n * Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.\n *\n * Uses `json_to_recordset()` so the batch is a single JSON parameter: the\n * statement text stays constant regardless of batch size, avoiding the\n * unbounded prepared-statement variants (and Postgres memory growth) that a\n * multi-row INSERT would produce. See rocicorp/mono#3511.\n *\n * Returns (and updates) {@link PendingTransaction.lastFlush}; a no-op when the\n * batch is empty.\n */\n #flushChangeLog(tx: PendingTransaction): Promise<unknown> | undefined {\n const {batch} = tx;\n if (batch.length === 0) {\n return tx.lastFlush;\n }\n tx.batch = [];\n tx.lastFlush = tx.pool.process(sql => [\n sql`\n INSERT INTO ${this.#cdc('changeLog')} (\"watermark\", \"pos\", \"change\", \"precommit\")\n SELECT \"watermark\", \"pos\", \"change\"::json, \"precommit\"\n FROM json_to_recordset(${batch}) AS x(\n \"watermark\" TEXT,\n \"pos\" INT8,\n \"change\" TEXT,\n \"precommit\" TEXT\n )`,\n ]);\n return tx.lastFlush;\n }\n\n #stopped = promiseVoid;\n\n /**\n * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n * Once {@link run()} completes, it can be called again.\n */\n async run() {\n assert(!this.#running, `storer is already running`);\n\n const {promise: stopped, resolve: signalStopped} = resolver();\n this.#running = true;\n this.#stopped = stopped;\n\n this.#lc.info?.('starting storer');\n this.#progressMonitor.start(); // Note: This is stopped in stop()\n let err: unknown;\n try {\n await this.#processQueue();\n } catch (e) {\n err = e; // used in finally\n throw e;\n } finally {\n // Release any pending backpressure so the upstream can proceed\n if (this.#readyForMore !== null) {\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n this.#cancelQueueEntries(\n this.#queue.drain().filter(entry => entry !== undefined),\n err,\n );\n this.#running = false;\n signalStopped();\n this.#lc.info?.('storer stopped');\n }\n }\n\n #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n if (queue.length === 0) {\n return;\n }\n this.#lc.info?.(\n `canceling ${queue.length} entries from the changeLog queue`,\n );\n const err = e instanceof Error ? e : new AbortError('server shutting down');\n for (const entry of queue) {\n if (entry === 'stop') {\n continue;\n }\n const type = entry[0];\n switch (type) {\n case 'subscriber': {\n // Disconnect subscribers waiting to be caught up so that they can\n // reconnect and try again.\n const {subscriber} = entry[1];\n this.#lc.info?.(`disconnecting ${subscriber.id}`);\n subscriber.fail(err);\n break;\n }\n }\n }\n }\n\n async #processQueue() {\n let tx: PendingTransaction | SkippedTransaction | null = null;\n let msg: QueueEntry | false;\n\n // Track the progress of each (previous and next) queue entry before it is\n // processed in the loop.\n let lastTaskDone: TaskDoneFn | undefined;\n const nextMessage = async () => {\n lastTaskDone?.();\n lastTaskDone = undefined;\n const entry = await this.#queue.dequeue();\n if (entry !== 'stop') {\n lastTaskDone = this.#progressMonitor.trackTask({\n task: 'queue-entry',\n // Note: TaskKeys are logged, so only include entry[0] (the type) to\n // avoid logging application data.\n type: entry[0],\n });\n }\n return entry;\n };\n\n const catchupQueue: SubscriberAndMode[] = [];\n try {\n while ((msg = await nextMessage()) !== 'stop') {\n const [msgType] = msg;\n switch (msgType) {\n case 'ready': {\n const signalReady = msg[1];\n signalReady();\n continue;\n }\n case 'subscriber': {\n const subscriber = msg[1];\n if (tx) {\n catchupQueue.push(subscriber); // Wait for the current tx to complete.\n } else {\n await this.#startCatchup([subscriber]); // Catch up immediately.\n }\n continue;\n }\n case 'status': {\n const pending = this.#pendingOwnership;\n if (pending && !tx && msg[1] >= pending.head) {\n const pool = new TransactionPool(this.#lc, {\n mode: Mode.READ_COMMITTED,\n });\n pool.run(this.#inserter);\n const assumed = await this.#tryAssumeOwnership(\n pool,\n msg[1],\n false,\n );\n if (assumed === null) {\n pool.abort();\n await pool.done();\n } else {\n pool.setDone();\n await pool.done();\n this.#ownershipAssumed(assumed);\n }\n }\n continue;\n }\n case 'abort': {\n if (tx && !tx.skipped) {\n tx.pool.abort();\n await tx.pool.done();\n }\n tx = null;\n continue;\n }\n }\n // msgType === 'change'\n this.#approximateQueuedBytes -= msg[2].length;\n tx = await this.#processChange(tx, msg);\n // Released once the change has been processed (e.g. flushed), so that\n // the upstream is held back by the progress of the change log.\n this.#maybeReleaseBackPressure();\n if (tx === null) {\n // Before beginning the next transaction, open a READONLY snapshot to\n // concurrently catchup subscribers that were waiting for the\n // transaction to end.\n await this.#startCatchup(catchupQueue.splice(0));\n }\n }\n // Subscribers waiting for a transaction that was interrupted (i.e.\n // aborted before stopping) are disconnected so that they can reconnect,\n // like those that are still in the queue.\n this.#cancelQueueEntries(\n catchupQueue.splice(0).map(sub => ['subscriber', sub]),\n undefined,\n );\n } catch (e) {\n catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n }\n\n /**\n * Processes a change from the queue, i.e. stores it in the change log as\n * part of the current transaction (or skips it, see\n * {@link assumeOwnershipAtHead}).\n *\n * @returns The transaction that is open after the change, or `null` if the\n * change ended (or was outside of) a transaction.\n */\n async #processChange(\n tx: PendingTransaction | SkippedTransaction | null,\n msg: Extract<QueueEntry, ['change', ...unknown[]]>,\n ): Promise<PendingTransaction | SkippedTransaction | null> {\n const [_, watermark, json, storedChange, change] = msg;\n const tag = change?.tag;\n\n if (tag === 'begin') {\n assert(!tx, 'received BEGIN in the middle of a transaction');\n const pending = this.#pendingOwnership;\n if (pending) {\n // Backfill transactions are neither stored nor used to assume\n // ownership. See assumeOwnershipAtHead().\n if (change.skipAck) {\n pending.skippedBackfill = true;\n return {skipped: true, ack: false};\n }\n if (watermark <= pending.head) {\n return {skipped: true, ack: true};\n }\n }\n const pool = new TransactionPool(\n this.#lc.withContext('watermark', watermark),\n {mode: Mode.READ_COMMITTED},\n );\n pool.run(this.#inserter);\n\n const {promise, resolve, reject} = resolver<ReplicationOwner>();\n void promise.catch(() => {}); // handle rejections before the await\n\n const assumed = pending\n ? await this.#tryAssumeOwnership(pool, watermark, true)\n : undefined;\n if (assumed === null) {\n // Stored by the previous owner in the meantime.\n pool.abort();\n await pool.done();\n return {skipped: true, ack: true};\n }\n if (pending) {\n // Ownership was assumed in this transaction.\n resolve({owner: this.#taskID});\n } else {\n // Acquire a lock on the replicationState row to detect and/or prevent\n // a concurrent ownership change.\n void pool.process(tx => {\n tx<ReplicationOwner[]> /*sql*/ `\n SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n ([result]) => resolve(result),\n reject,\n );\n return [];\n });\n }\n\n tx = {\n pool,\n preCommitWatermark: watermark,\n pos: 0,\n startingReplicationState: promise,\n ack: !change.skipAck,\n batch: [],\n lastFlush: undefined,\n ownershipAssumed: assumed,\n };\n } else if (tx?.skipped) {\n if (tag === 'commit' && tx.ack) {\n this.#onCommitted(['commit', change, {watermark}]);\n }\n return tag === 'commit' || tag === 'rollback' ? null : tx;\n } else {\n assert(tx, () => `received change outside of transaction: ${json}`);\n tx.pos++;\n }\n\n const entry: ChangeLogRow = {\n watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n pos: tx.pos,\n // For backwards compatibility, only the change message is stored\n // in the cdc changeLog. It was serialized separately with the\n // downstream envelope, so no full-message scan is needed here.\n change: storedChange,\n };\n\n if (change !== null && isSchemaChange(change)) {\n // Schema changes carry backfill / table-metadata statements that\n // must be applied in stream order relative to the changeLog rows.\n // Flush any buffered rows first, then write this row together with\n // its metadata statements as a single unit (preserving the previous\n // per-change ordering for schema changes).\n await this.#flushChangeLog(tx);\n tx.lastFlush = tx.pool.process(sql => [\n sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n ...this.#trackBackfillMetadata(sql, change),\n ]);\n } else {\n // Accumulate plain changeLog rows (begin, data changes, commit) and\n // write them as a single multi-row INSERT. Collapsing the per-change\n // single-row INSERTs into batches is the dominant cost reduction for\n // large transactions, where the previous one-statement-per-change\n // path dominated the upstream replication lag.\n tx.batch.push(entry);\n if (tx.batch.length >= this.#changeLogBatchSize) {\n // Bound pipeline depth (and thus memory) by awaiting the previous\n // flush before issuing the next. This is the batched analog of the\n // previous per-100-statement backpressure await, and likewise\n // guards against memory blowup on very large transactions.\n const prevFlush = tx.lastFlush;\n void this.#flushChangeLog(tx);\n await prevFlush;\n }\n }\n if (tag === 'commit') {\n // Flush any remaining buffered changeLog rows (including this commit\n // row) before updating the replication state, so the state update is\n // ordered after all changeLog inserts for this transaction.\n void this.#flushChangeLog(tx);\n\n const {owner} = await tx.startingReplicationState;\n if (owner !== this.#taskID) {\n // Ownership change reflected in the replicationState read in 'begin'.\n tx.pool.fail(\n new AbortError(`changeLog ownership has been assumed by ${owner}`),\n );\n } else {\n // Update the replication state.\n const lastWatermark = watermark;\n void tx.pool.process(tx => [\n tx`\n UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n ]);\n tx.pool.setDone();\n }\n\n await tx.pool.done();\n\n const {ownershipAssumed, ack} = tx;\n tx = null;\n if (ownershipAssumed) {\n this.#ownershipAssumed(ownershipAssumed);\n }\n // ACK the LSN to the upstream Postgres.\n if (ack) {\n this.#onCommitted(['commit', change, {watermark}]);\n }\n } else if (tag === 'rollback') {\n // Aborted transactions are not stored in the changeLog.\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n }\n return tx;\n }\n\n async #startCatchup(subs: SubscriberAndMode[]) {\n const numCatchups = subs.length;\n if (numCatchups === 0) {\n return;\n }\n\n const lc = this.#lc.withContext('pool', 'catchup');\n const {init, cleanup, snapshotID} = sharedSnapshot();\n const reader = new TransactionPool(lc, {\n mode: Mode.READONLY,\n init,\n cleanup,\n initialWorkers: subs.length,\n });\n\n // A dynamic connection pool is created for catchup, sized to the number\n // of subscribers being caught up.\n const catchupConns = this.#makeConnectionPool(\n 'subscriber-catchup',\n numCatchups,\n );\n reader.run(catchupConns);\n\n let lastWatermark: string | undefined;\n const catchupSnapshotted = this.#progressMonitor.trackTask({\n task: 'capture-catchup-snapshot',\n subscribers: subs.map(({subscriber: s}) => s.id),\n });\n try {\n // Ensure that the transaction has started (and is thus holding a snapshot\n // of the database) before continuing on to commit more changes. This is\n // done by performing a single read on the db, which determines the\n // snapshot for the REPEATABLE_READ transaction.\n [{lastWatermark}] = await reader.processReadTask(\n sql => sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}\n `,\n );\n lc.info?.(\n `snapshotted db at ${lastWatermark} (${await snapshotID}) ` +\n `to catchup ${numCatchups} subscriber(s)`,\n );\n } catch (e) {\n subs.map(({subscriber}) => subscriber.fail(e));\n // Nothing will run the catchup tasks that normally finalize the pool\n // (below), so tear it down here. Otherwise its workers (each holding an\n // open READ ONLY transaction) and the connection pool are leaked.\n reader.abort();\n void catchupConns.end().catch(() => {});\n throw e;\n } finally {\n catchupSnapshotted();\n }\n\n // Run the actual catchup queries in the background. Errors are handled in\n // #catchup() by disconnecting the associated subscriber.\n void Promise.allSettled(\n subs.map(sub =>\n this.#catchup(\n lc.withContext('subscriber', sub.subscriber.id),\n sub,\n lastWatermark,\n reader,\n ),\n ),\n ).finally(() => {\n reader.setDone();\n void catchupConns.end().catch(() => {});\n });\n }\n\n async #catchup(\n lc: LogContext,\n {subscriber: sub, mode}: SubscriberAndMode,\n lastWatermark: string,\n reader: TransactionPool,\n ) {\n let entriesReceived = 0;\n let catchupProgressed = this.#progressMonitor.trackTask({\n task: 'catchup',\n subscriber: sub.id,\n entriesReceived,\n });\n try {\n lc.info?.(`starting catchup`);\n await reader.processReadTask(async tx => {\n lc.info?.(`catching up`);\n const start = Date.now();\n\n // When starting from initial-sync, there won't be a change with a watermark\n // equal to the replica version. This is the empty changeLog scenario.\n let watermarkFound = sub.watermark === this.#replicaVersion;\n let count = 0;\n let lastBatchConsumed: Promise<unknown> | undefined;\n\n for await (const entries of tx<ChangeLogEntry[]> /*sql*/ `\n SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc('changeLog')}\n WHERE watermark >= ${sub.watermark}\n AND watermark <= ${lastWatermark}\n ORDER BY watermark, pos`.cursor(2000)) {\n catchupProgressed();\n entriesReceived += entries.length;\n\n try {\n // Wait for the last batch of entries to be consumed by the\n // subscriber before sending down the current batch. This pipelining\n // allows one batch of changes to be received from the change-db\n // while the previous batch of changes are sent to the subscriber,\n // resulting in flow control that caps the number of changes\n // referenced in memory to 2 * batch-size.\n const start = performance.now();\n await lastBatchConsumed;\n const elapsed = performance.now() - start;\n if (lastBatchConsumed) {\n lc[elapsed > 100 ? 'info' : 'debug']?.(\n `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n );\n }\n\n for (const entry of entries) {\n if (entry.watermark === sub.watermark) {\n // This should be the first entry.\n // Catchup starts from *after* the watermark.\n watermarkFound = true;\n } else if (watermarkFound) {\n lastBatchConsumed = sub.catchup(\n reconstructWatermarkedChange(entry),\n );\n count++;\n } else if (mode === 'backup') {\n throw new AutoResetSignal(\n `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n );\n } else {\n lc.warn?.(\n `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n );\n sub.close(\n ErrorType.WatermarkTooOld,\n `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n );\n return;\n }\n }\n } finally {\n // Track the db's read of the next chunk of changes.\n catchupProgressed = this.#progressMonitor.trackTask({\n task: 'catchup',\n subscriber: sub.id,\n entriesReceived,\n });\n }\n }\n catchupProgressed();\n\n if (watermarkFound) {\n await lastBatchConsumed;\n lc.info?.(\n `caught up ${sub.id} with ${count} changes (${\n Date.now() - start\n } ms)`,\n );\n } else {\n // The subscriber is ahead of the latest durable changeLog entry\n // (lastWatermark). This can legitimately happen: changes are\n // forwarded to subscribers (the backup replica and view-syncers)\n // concurrently with — and can outrun — the durable store, so a\n // replica may briefly lead the change DB after the storer falls\n // behind or the change-streamer restarts. No catchup is possible or\n // needed; once the change DB catches back up, forwarding resumes and\n // the subscriber dedups any watermarks it already has. Unlike the\n // AutoResetSignal / WatermarkTooOld cases above, this is not a gap in\n // replication history, so the subscriber is simply marked caught up.\n lc.warn?.(\n `subscriber ${sub.id} at watermark ${sub.watermark} is ahead of ` +\n `the latest durable watermark ${lastWatermark}; waiting for the ` +\n `change DB to catch up`,\n );\n }\n // Start draining messages buffered during catchup. The returned promise\n // is intentionally not awaited here: while the drain is in progress,\n // new sends keep appending to the subscriber backlog and inherit its\n // byte-based backpressure.\n void sub.setCaughtUp();\n });\n } catch (err) {\n lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n if (err instanceof AutoResetSignal) {\n await markResetRequired(this.#db, this.#shard);\n this.#onFatal(err);\n }\n sub.fail(err);\n } finally {\n catchupProgressed();\n }\n }\n\n /**\n * Returns the db statements necessary to track backfill and table metadata\n * presented in the `change`, if any.\n *\n * This is the Postgres interpreter of {@link cookieOps}, which is also\n * interpreted against the SQLite change log by `ChangeLogCookieWriter`. The\n * two stores have to agree on every transition forever, and the failure mode\n * if they drift is a backfill that is silently never re-requested — so the\n * decision of what a schema change *means* lives in one place and only its\n * transport lives here.\n */\n #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n return cookieOps(change).flatMap(op => this.#cookieStmts(sql, op));\n }\n\n #cookieStmts(sql: PostgresTransaction, op: CookieOp): PendingQuery<Row[]>[] {\n switch (op.op) {\n case 'upsert-metadata':\n return [this.#upsertTableMetadataStmt(sql, op.table, op.metadata)];\n\n case 'upsert-backfill':\n return [\n this.#upsertColumnBackfillStmt(sql, op.table, op.column, op.backfill),\n ];\n\n case 'rename-table': {\n const {old} = op;\n const row = {schema: op.new.schema, table: op.new.name};\n return [\n sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n ];\n }\n\n case 'drop-table': {\n const {schema, name} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('tableMetadata')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n ];\n }\n\n case 'rename-column': {\n const {schema, name: table} = op.table;\n return [\n sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${op.new}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${op.old}`,\n ];\n }\n\n case 'drop-column': {\n const {schema, name} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${op.column}`,\n ];\n }\n\n case 'complete-backfill': {\n const {schema, name: table} = op.table;\n return [\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql([...op.columns])}`,\n ];\n }\n\n default:\n unreachable(op);\n }\n }\n\n #upsertTableMetadataStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n metadata: TableMetadata,\n ) {\n const row: TableMetadataRow = {schema, table, metadata};\n return sql`\n INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\")\n DO UPDATE SET ${sql(row)};\n `;\n }\n\n #upsertColumnBackfillStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n column: string,\n backfill: BackfillID,\n ) {\n const row: BackfillingColumn = {schema, table, column, backfill};\n return sql`\n INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\", \"column\")\n DO UPDATE SET ${sql(row)};\n `;\n }\n\n /**\n * Waits until all currently queued entries have been processed.\n * This is only used in tests.\n */\n async allProcessed() {\n if (this.#running) {\n const {promise, resolve} = resolver();\n this.#queue.enqueue(['ready', resolve]);\n await promise;\n }\n }\n\n /**\n * Stops the storer and waits up to a drain timeout for entries to drain,\n * throwing an exception if it doesn't drain in time, in order to abort\n * the server when PG (or the connection to it) appears to be wedged.\n */\n async stop() {\n if (this.#running) {\n this.#progressMonitor.stop();\n this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n this.abort(); // for cleanliness, abort any open transactions\n this.#queue.enqueue('stop');\n }\n if (\n (await orTimeout(this.#stopped, this.#drainTimeoutMs)) === 'timed-out'\n ) {\n throw new AbortError(\n `changeLog did not drain within ${this.#drainTimeoutMs}ms`,\n );\n }\n }\n}\n\nexport class PurgeLock {\n readonly #lc: LogContext;\n readonly #tx: TransactionPool;\n readonly replicaVersion: string;\n readonly minWatermark: string;\n /**\n * The head of the change log (i.e. its `lastWatermark`) when the lock was\n * acquired. The change log is resumed from its head, which only advances,\n * so a stream cannot (correctly) be started from before it.\n */\n readonly headWatermark: string;\n\n constructor(\n lc: LogContext,\n tx: TransactionPool,\n replicaVersion: string,\n watermark: string,\n headWatermark: string,\n ) {\n this.#lc = lc;\n this.#tx = tx;\n this.replicaVersion = replicaVersion;\n this.minWatermark = watermark;\n this.headWatermark = headWatermark;\n }\n\n #released = false;\n\n async release() {\n if (this.#released) {\n return;\n }\n this.#released = true;\n this.#tx.setDone();\n await this.#tx\n .done()\n .catch(e => this.#lc.warn?.(`error from purge-lock release`, e));\n this.#lc.info?.(`released purge lock on ${this.minWatermark}`);\n }\n}\n\nexport class PurgeLocker {\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #db: PostgresDB;\n\n constructor(lc: LogContext, shard: ShardID, db: PostgresDB) {\n this.#lc = lc.withContext('component', 'purge-locker');\n this.#shard = shard;\n this.#db = db;\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n /**\n * Locks the oldest row of the change log, preventing it from being purged.\n *\n * @param slotWatermark The position of the replication slot from which the\n * change log is to be resumed (as a watermark). If specified and the\n * head of the change log is behind it, the change log may be missing\n * transactions that the slot will not stream, in which case nothing is\n * locked and `'behind-slot'` is returned. Conversely, a head at or past\n * the slot's position means that the stream (which resumes from the\n * head) will not skip anything.\n * @returns `null` if the change log is empty.\n */\n acquire(): Promise<PurgeLock | null>;\n acquire(\n slotWatermark?: string | undefined,\n ): Promise<PurgeLock | null | 'behind-slot'>;\n async acquire(\n slotWatermark?: string,\n ): Promise<PurgeLock | null | 'behind-slot'> {\n const tx = new TransactionPool(this.#lc, {mode: Mode.READ_COMMITTED}).run(\n this.#db,\n );\n const [state] = await tx.processReadTask(\n sql => sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n );\n // An uninitialized change log (i.e. without a head) is empty, and thus\n // behind any replication slot.\n const lastWatermark = state?.lastWatermark;\n if (\n slotWatermark !== undefined &&\n (lastWatermark === undefined || lastWatermark < slotWatermark)\n ) {\n this.#lc.info?.(\n `changeLog@${lastWatermark ?? '(uninitialized)'} is behind the replication slot@${slotWatermark}`,\n );\n tx.setDone();\n await tx.done();\n return 'behind-slot';\n }\n if (lastWatermark === undefined) {\n this.#lc.info?.(`changeLog is uninitialized. No rows to purge-lock.`);\n tx.setDone();\n await tx.done();\n return null;\n }\n const row = await tx.processReadTask(\n sql => sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR SHARE \n `,\n );\n if (row.length === 0) {\n this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);\n tx.setDone();\n await tx.done();\n return null;\n }\n const [{watermark}] = row;\n const [{replicaVersion}] = await tx.processReadTask(\n sql => sql<{replicaVersion: string}[]>`\n SELECT \"replicaVersion\" FROM ${this.#cdc('replicationConfig')}\n `,\n );\n this.#lc.info?.(\n `locked watermark ${watermark} from being purged from replica@${replicaVersion}`,\n );\n return new PurgeLock(\n this.#lc,\n tx,\n replicaVersion,\n watermark,\n lastWatermark,\n );\n }\n}\n\n// A TaskKey, keyed by identity, representing an db task monitored by the\n// ProgressMonitor. The contents are stringified in error messages if the\n// task fails to make progress for more than a failure threshold interval.\ntype TaskKey = JSONObject;\n\ntype TaskDoneFn = () => void;\n\n/**\n * Periodic monitor for detecting the pathological condition in which a db\n * query hangs forever because of a connection problem, such as a half-closed\n * socket.\n *\n * Use this for db calls that recur or accumulate progress within a single\n * logical operation on the main storer loop (e.g. the queue-processing loop,\n * or a multi-batch catchup read), where \"no progress for N ms\" is the\n * meaningful signal. For a one-off db call -- including ones made before\n * {@link Storer.run} starts this monitor's polling -- use a plain timeout\n * (see `Storer.#withTimeout`) instead.\n *\n * The failure handler should stop the server. The ProgressMonitor will\n * assume this and stop itself after calling it.\n *\n * Internal to the Storer, but exported for testing.\n */\nexport class ProgressMonitor {\n readonly #lc: LogContext;\n readonly #failureThresholdMs;\n readonly #onFailure: (err: Error) => void;\n readonly #taskUpdates = new Map<TaskKey, Date>();\n\n #timer: NodeJS.Timeout | undefined;\n\n constructor(\n lc: LogContext,\n failureThresholdMs: number,\n onFailure: (err: Error) => void,\n ) {\n this.#lc = lc.withContext('component', 'storer-progress-monitor');\n this.#failureThresholdMs = failureThresholdMs;\n this.#onFailure = onFailure;\n }\n\n start() {\n clearInterval(this.#timer);\n this.#timer = setInterval(\n () => this.checkTaskProgress(new Date()),\n this.#failureThresholdMs,\n ).unref();\n }\n\n // Called by an internal periodic timer, but exported for testing.\n checkTaskProgress(now: Date) {\n for (const [task, lastUpdate] of this.#taskUpdates.entries()) {\n if (now.getTime() - lastUpdate.getTime() >= this.#failureThresholdMs) {\n this.#lc.error?.(\n `Last task update was more than ${this.#failureThresholdMs}ms ago`,\n {task, lastUpdate},\n );\n this.#onFailure(\n new Error(\n `Task failed to progress for over ${this.#failureThresholdMs}ms: ${BigIntJSON.stringify(task)}`,\n ),\n );\n this.stop();\n return;\n }\n }\n }\n\n /**\n * Starts a task to be tracked.\n *\n * @return a callback to invoke when the task is done.\n */\n trackTask(task: TaskKey, now = new Date()): TaskDoneFn {\n this.#taskUpdates.set(task, now);\n return () => {\n this.#taskUpdates.delete(task);\n };\n }\n\n stop() {\n clearInterval(this.#timer);\n this.#taskUpdates.clear();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA0JA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkD5D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CAKA;CACA;CACA;CAEA;CACA;CACA;CACA;;;;;CAMA,oBAQW;CACX,SAAkB,IAAI,MAAkB;CACxC;CACA;CACA;CACA;CACA;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,YACA,mBACA,aACA,SACA,EACE,iCACA,oBACA,oBACA,iBAAiB,OAEnB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKC,SAAS;EACd,KAAKC,UAAU;EACf,KAAKC,oBAAoB;EACzB,KAAKC,qBAAqB;EAC1B,KAAKC,sBAAsB;EAC3B,KAAKC,MAAM,WAAW,sBAAsB,CAAC;EAC7C,KAAKC,YAAY,WAAW,uBAAuB,CAAC;EACpD,KAAKC,UAAU,WAAW,qBAAqB,CAAC;EAChD,KAAKC,qBAAqB;EAC1B,KAAKC,kBAAkB,kBAAkB;EACzC,KAAKC,eAAe;EACpB,KAAKC,WAAW;EAChB,KAAKG,sBAAsB;EAC3B,KAAKC,sBAAsB,KAAK,IAAI,GAAG,kBAAkB;EACzD,KAAKC,kBAAkB;EACvB,KAAKC,mBAAmB,IAAI,gBAC1B,IACA,oBACA,OACF;EAEA,MAAM,YAAY,kBAAkB;EACpC,KAAKJ,+BACF,UAAU,kBAAkB,UAAU,kBACvC;EAEF,KAAKd,IAAI,OACP,gBAAgB,KAAKc,8BAA8B,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,CAAC,EAAE,iCAE/E,EAAC,UAAS,CACZ;CACF;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKR,IAAI,GAAG,UAAU,KAAKL,MAAM,EAAE,GAAG,OAAO;CACtD;;;;;;;;;;;;;;;CAgBA,MAAMkB,aAAgB,MAAc,SAAiC;EACnE,MAAM,SAAS,MAAM,UAAU,SAAS,KAAKJ,mBAAmB;EAChE,IAAI,WAAW,aAAa;GAC1B,QAAa,YAAY,CAAC,CAAC;GAC3B,MAAM,IAAI,WACR,GAAG,KAAK,2BAA2B,KAAKA,oBAAoB,GAC9D;EACF;EACA,OAAO;CACT;CAEA,gBAAgB;EACd,MAAM,eAAe,KAAKZ;EAC1B,MAAM,gBAAgB,KAAKC;EAE3B,OAAO,kBAAkB,OACrB,eACA,GAAG,cAAc,KAAK;CAC5B;;;;;;;CAQA,wBACE,QACyC;EACzC,OAAO,sBACL,KAAKJ,KACL,KAAKM,KACL,KAAKL,QACL;GAAC,GAAG;GAAQ,OAAO,KAAKC;GAAS,cAAc,KAAKkB,cAAc;EAAC,GAGnE,KAAKC,mBAAmB,aAAa,KAAA,CACvC;CACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkCA,MAAM,sBAAsB,UAA6B;EAKvD,MAAM,wBACJ,KAAKrB,KACL,KAAKM,KACL,KAAKG,oBACL,KAAKR,QACL,SAAS,WACT,SAAS,aAAa,KAAA,GACtB,KAAA,GACA,SAAS,qBACX;EACA,KAAKoB,oBAAoB;GACvB,GAAG;GACH,MAAM;GACN,iBAAiB;GACjB,OAAO,YAAY,IAAI;EACzB;CACF;;;;;CAMA,IAAI,mBAA4B;EAC9B,OAAO,KAAKA,sBAAsB;CACpC;;;;;;CAOA,OAAO,WAAmB;EACxB,IAAI,KAAKA,mBACP,KAAKR,OAAO,QAAQ,CAAC,UAAU,SAAS,CAAC;CAE7C;;;;;;;;;;;CAYA,MAAMS,oBACJ,IACA,WACA,UACkC;EAClC,MAAM,UAAU,KAAK,KAAKD,iBAAiB;EAC3C,MAAM,CAAC,SAAS,MAAM,GAAG,iBACvB,QAAO,GAA8B;sCACL,KAAKE,KAAK,kBAAkB,EAAE,YAChE;EACA,MAAM,OAAO,MAAM;EACnB,QAAQ,OAAO,OAAO,QAAQ,OAAO,OAAO,QAAQ;EACpD,IAAI,WAAW,QAAQ,YAAY,OAAO,WACxC,OAAO;EAET,MAAM,QAAQ,KAAKrB;EACnB,MAAM,eAAe,KAAKkB,cAAc;EACxC,GAAQ,SAAQ,QAAO,CACrB,GAAG,UAAU,KAAKG,KAAK,kBAAkB,EAAE,OAAO,IAAI;GAAC;GAAO;EAAY,CAAC,GAC7E,CAAC;EAED,MAAM,CAAC,EAAC,iBAAgB,MAAM,GAAG,iBAC/B,QAAO,GAA6B;uCACH,KAAKA,KAAK,aAAa,EAAE,mBAC5D;EACA,KAAKvB,IAAI,OACP,yBAAyB,aAAa,yBAAyB,MACjE;EACA,OAAO;GAAC;GAAM;EAAW;CAC3B;CAEA,kBAAkB,EAAC,MAAM,eAAgC;EACvD,MAAM,EAAC,WAAW,gBAAgB,iBAAiB,UAAS,KAC1D,KAAKqB,iBACP;EACA,KAAKA,oBAAoB;EACzB,KAAKrB,IAAI,OACP,0CAA0C,KAAK,UACzC,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC,EAAE,IAC9C;EAIA,WAAgB,QAAQ;EACxB,IAAI,eAAe,iBACjB,eAAe,6CAA6C;CAEhE;;;;;;;;CASA,MAAM,kBAAkB;EACtB,MAAM,KAAK,KAAKM;EAChB,MAAM,QAAQ,KAAKJ;EACnB,MAAM,sBAAsB,KAAKkB,cAAc;EAC/C,KAAKpB,IAAI,OAAO,yBAAyB,qBAAqB;EAC9D,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAKmB,aACT,oBACA,EAAE,UAAU,KAAKI,KAAK,kBAAkB,EAAE,OAAO,GAAG;GAAC;GAAO,cAAc;EAAmB,CAAC,GAChG;EACA,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,KAAKvB,IAAI,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,KAC1D;CACF;;;;;;;;;;;;;;;;;;CAmBA,MAAM,yCAIH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,QAAQ,eAAe,eAC/C,MAAM,KAAKmB,aACT,qBACA,MACE,KAAKb,MACL,QAAO;GACL,GAA8B;sCACJ,KAAKiB,KAAK,kBAAkB;GAOtD,GAAG;;;;;;;;;iBASE,KAAKA,KAAK,aAAa,EAAE;sBACpB,KAAKA,KAAK,eAAe,EAAE;;;;GAWrC,GAA0B;oDACc,KAAKA,KAAK,eAAe,EAAE;;GAGnE,GAAqB;8DAC6B,KAAKA,KAAK,aAAa,EAAE;;EAE7E,GACA,EAAC,MAAM,SAAa,CACtB,CACF;EAEF,OAAO;GACL;GAGA,kBAAkB,KAAKF,oBACnB,CAAC,IACD,MAAQ,QAAQ,sBAAsB;GAC1C,SAAS;IACP,eAAe,CAAC,GAAG,aAAa;IAChC,aAAa,CAAC,GAAG,WAAW;GAC9B;EACF;CACF;CAEA,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,KAAKF,aAClC,qBACA,KAAKb,GAA6C;qDACH,KAAKiB,KAAK,WAAW,GACtE;EACA,OAAO;CACT;;;;;;CAOA,MAAM,mBAGH;EACD,MAAM,CAAC,UAAU,MAAM,KAAKJ,aAC1B,sBACA,KAAKb,GAAoE;;sCAEzC,KAAKiB,KAAK,WAAW,EAAE;uCACtB,KAAKA,KAAK,kBAAkB,EAAE,qBACjE;EACA,OAAO;CACT;;;;;CAMA,MAAM,qBACJ,gBACA,kBACA,OACmB;EAQnB,QAAO,MAPY,KAAKjB,GAAmC;8BACjC,KAAKiB,KAAK,WAAW,EAAE;;2BAE1B,eAAe;4BACd,iBAAiB;;eAE9B,SACC,KAAK,EAAC,gBAAe,SAAS;CAC5C;;;;;;;;CASA,iBACE,gBACA,kBACA,YAAY,KACqB;EACjC,OACE,OAAO,cAAc,SAAS,KAAK,YAAY,GAC/C,gEACF;EACA,OAAO,KAAKjB,GAA8B;kEACoB,KAAKiB,KAAK,WAAW,EAAE;2BAC9D,eAAe;4BACd,iBAAiB;gCACb,OAAO,SAAS;CAG9C;CAMA,mBAAmB,WAAoC;EACrD,OAAO,MAAM,KAAKf,SAAS,OAAM,QAAO;GAGtC,MAAM,GAA0B;kCACJ,KAAKe,KAAK,WAAW,EAAE;;;;GASnD,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;;;;;;;;iBASvC,KAAKA,KAAK,WAAW,EAAE;;wBAEhB,KAAKA,KAAK,WAAW,EAAE,qBAAqB,UAAU;;;;GAKxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;8BACvB,KAAKA,KAAK,kBAAkB,EAAE;GACtD,IAAI,UAAU,KAAKrB,SACjB,MAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,OAClF;GAEF,OAAO,OAAO,OAAO;EACvB,CAAC;CACH;;;;CAKA,MACE,WACA,MACA,aAAyC,oCACvC,IACF,GACA;EAIA,MAAM,EAAC,MAAM,QAAQ,iBAAgB;EACrC,KAAKsB,2BAA2B,KAAK;EAErC,MAAM,SAAS,KAAK;EACpB,KAAKX,OAAO,QAAQ;GAClB;GACA;GACA;GACA;GACA,aAAa,MAAM,IAAI,OAAO;EAChC,CAAC;EAED,OAAO;CACT;CAEA,QAAQ;EACN,KAAKA,OAAO,QAAQ,CAAC,OAAO,CAAC;CAC/B;CAEA,QAAQ,YAAwB,MAAsB;EACpD,KAAKA,OAAO,QAAQ,CAAC,cAAc;GAAC;GAAY;EAAI,CAAC,CAAC;CACxD;CAEA,gBAAuC;CAEvC,eAA0C;EACxC,IAAI,CAAC,KAAKY,UACR;EAEF,IACE,KAAKC,kBAAkB,QACvB,KAAKF,0BAA0B,KAAKV,6BACpC;GACA,KAAKd,IAAI,OACP,+BAA+B,KAAKa,OAAO,KAAK,EAAE,qBAAqB,KAAKW,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,2SAW9H;GACA,KAAKE,gBAAgB,SAAS;EAChC;EACA,OAAO,KAAKA,eAAe;CAC7B;CAEA,4BAA4B;EAC1B,IAAI,KAAKA,kBAAkB;OAGvB,KAAKF,0BACL,KAAKV,8BAA8B,IACnC;IACA,KAAKd,IAAI,OACP,gCAAgC,KAAKa,OAAO,KAAK,EAAE,qBAAqB,KAAKW,0BAA0B,QAAQ,GAAG,QAAQ,CAAC,EAAE,KAC/H;IACA,KAAKE,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;;CAEJ;;;;;;;;;;;;CAaA,gBAAgB,IAAsD;EACpE,MAAM,EAAC,UAAS;EAChB,IAAI,MAAM,WAAW,GACnB,OAAO,GAAG;EAEZ,GAAG,QAAQ,CAAC;EACZ,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG;sBACa,KAAKH,KAAK,WAAW,EAAE;;mCAEV,MAAM;;;;;YAMrC,CAAC;EACD,OAAO,GAAG;CACZ;CAEA,WAAW;;;;;CAMX,MAAM,MAAM;EACV,OAAO,CAAC,KAAKE,UAAU,2BAA2B;EAElD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,SAAS;EAC5D,KAAKA,WAAW;EAChB,KAAKE,WAAW;EAEhB,KAAK3B,IAAI,OAAO,iBAAiB;EACjC,KAAKkB,iBAAiB,MAAM;EAC5B,IAAI;EACJ,IAAI;GACF,MAAM,KAAKU,cAAc;EAC3B,SAAS,GAAG;GACV,MAAM;GACN,MAAM;EACR,UAAU;GAER,IAAI,KAAKF,kBAAkB,MAAM;IAC/B,KAAKA,cAAc,QAAQ;IAC3B,KAAKA,gBAAgB;GACvB;GACA,KAAKG,oBACH,KAAKhB,OAAO,MAAM,EAAE,QAAO,UAAS,UAAU,KAAA,CAAS,GACvD,GACF;GACA,KAAKY,WAAW;GAChB,cAAc;GACd,KAAKzB,IAAI,OAAO,gBAAgB;EAClC;CACF;CAEA,oBAAoB,OAAqB,GAAY;EACnD,IAAI,MAAM,WAAW,GACnB;EAEF,KAAKA,IAAI,OACP,aAAa,MAAM,OAAO,kCAC5B;EACA,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,sBAAsB;EAC1E,KAAK,MAAM,SAAS,OAAO;GACzB,IAAI,UAAU,QACZ;GAGF,QADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;KAC3B,KAAKA,IAAI,OAAO,iBAAiB,WAAW,IAAI;KAChD,WAAW,KAAK,GAAG;KACnB;IACF;GACF;EACF;CACF;CAEA,MAAM4B,gBAAgB;EACpB,IAAI,KAAqD;EACzD,IAAI;EAIJ,IAAI;EACJ,MAAM,cAAc,YAAY;GAC9B,eAAe;GACf,eAAe,KAAA;GACf,MAAM,QAAQ,MAAM,KAAKf,OAAO,QAAQ;GACxC,IAAI,UAAU,QACZ,eAAe,KAAKK,iBAAiB,UAAU;IAC7C,MAAM;IAGN,MAAM,MAAM;GACd,CAAC;GAEH,OAAO;EACT;EAEA,MAAM,eAAoC,CAAC;EAC3C,IAAI;GACF,QAAQ,MAAM,MAAM,YAAY,OAAO,QAAQ;IAC7C,MAAM,CAAC,WAAW;IAClB,QAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;MACxB,YAAY;MACZ;KACF;KACA,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;MACvB,IAAI,IACF,aAAa,KAAK,UAAU;WAE5B,MAAM,KAAKY,cAAc,CAAC,UAAU,CAAC;MAEvC;KACF;KACA,KAAK,UAAU;MACb,MAAM,UAAU,KAAKT;MACrB,IAAI,WAAW,CAAC,MAAM,IAAI,MAAM,QAAQ,MAAM;OAC5C,MAAM,OAAO,IAAI,gBAAgB,KAAKrB,KAAK,EACzC,MAAM,eACR,CAAC;OACD,KAAK,IAAI,KAAKO,SAAS;OACvB,MAAM,UAAU,MAAM,KAAKe,oBACzB,MACA,IAAI,IACJ,KACF;OACA,IAAI,YAAY,MAAM;QACpB,KAAK,MAAM;QACX,MAAM,KAAK,KAAK;OAClB,OAAO;QACL,KAAK,QAAQ;QACb,MAAM,KAAK,KAAK;QAChB,KAAKS,kBAAkB,OAAO;OAChC;MACF;MACA;KACF;KACA,KAAK;MACH,IAAI,MAAM,CAAC,GAAG,SAAS;OACrB,GAAG,KAAK,MAAM;OACd,MAAM,GAAG,KAAK,KAAK;MACrB;MACA,KAAK;MACL;IAEJ;IAEA,KAAKP,2BAA2B,IAAI,GAAG;IACvC,KAAK,MAAM,KAAKQ,eAAe,IAAI,GAAG;IAGtC,KAAKC,0BAA0B;IAC/B,IAAI,OAAO,MAIT,MAAM,KAAKH,cAAc,aAAa,OAAO,CAAC,CAAC;GAEnD;GAIA,KAAKD,oBACH,aAAa,OAAO,CAAC,EAAE,KAAI,QAAO,CAAC,cAAc,GAAG,CAAC,GACrD,KAAA,CACF;EACF,SAAS,GAAG;GACV,aAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GACzD,MAAM;EACR;CACF;;;;;;;;;CAUA,MAAMG,eACJ,IACA,KACyD;EACzD,MAAM,CAAC,GAAG,WAAW,MAAM,cAAc,UAAU;EACnD,MAAM,MAAM,QAAQ;EAEpB,IAAI,QAAQ,SAAS;GACnB,OAAO,CAAC,IAAI,+CAA+C;GAC3D,MAAM,UAAU,KAAKX;GACrB,IAAI,SAAS;IAGX,IAAI,OAAO,SAAS;KAClB,QAAQ,kBAAkB;KAC1B,OAAO;MAAC,SAAS;MAAM,KAAK;KAAK;IACnC;IACA,IAAI,aAAa,QAAQ,MACvB,OAAO;KAAC,SAAS;KAAM,KAAK;IAAI;GAEpC;GACA,MAAM,OAAO,IAAI,gBACf,KAAKrB,IAAI,YAAY,aAAa,SAAS,GAC3C,EAAC,MAAM,eAAmB,CAC5B;GACA,KAAK,IAAI,KAAKO,SAAS;GAEvB,MAAM,EAAC,SAAS,SAAS,WAAU,SAA2B;GAC9D,QAAa,YAAY,CAAC,CAAC;GAE3B,MAAM,UAAU,UACZ,MAAM,KAAKe,oBAAoB,MAAM,WAAW,IAAI,IACpD,KAAA;GACJ,IAAI,YAAY,MAAM;IAEpB,KAAK,MAAM;IACX,MAAM,KAAK,KAAK;IAChB,OAAO;KAAC,SAAS;KAAM,KAAK;IAAI;GAClC;GACA,IAAI,SAEF,QAAQ,EAAC,OAAO,KAAKpB,QAAO,CAAC;QAI7B,KAAU,SAAQ,OAAM;IACtB,EAA+B;kCACP,KAAKqB,KAAK,kBAAkB,EAAE,aAAa,MAChE,CAAC,YAAY,QAAQ,MAAM,GAC5B,MACF;IACA,OAAO,CAAC;GACV,CAAC;GAGH,KAAK;IACH;IACA,oBAAoB;IACpB,KAAK;IACL,0BAA0B;IAC1B,KAAK,CAAC,OAAO;IACb,OAAO,CAAC;IACR,WAAW,KAAA;IACX,kBAAkB;GACpB;EACF,OAAO,IAAI,IAAI,SAAS;GACtB,IAAI,QAAQ,YAAY,GAAG,KACzB,KAAKZ,aAAa;IAAC;IAAU;IAAQ,EAAC,UAAS;GAAC,CAAC;GAEnD,OAAO,QAAQ,YAAY,QAAQ,aAAa,OAAO;EACzD,OAAO;GACL,OAAO,UAAU,2CAA2C,MAAM;GAClE,GAAG;EACL;EAEA,MAAM,QAAsB;GAC1B,WAAW,QAAQ,WAAW,YAAY,GAAG;GAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;GACtD,KAAK,GAAG;GAIR,QAAQ;EACV;EAEA,IAAI,WAAW,QAAQ,eAAe,MAAM,GAAG;GAM7C,MAAM,KAAKuB,gBAAgB,EAAE;GAC7B,GAAG,YAAY,GAAG,KAAK,SAAQ,QAAO,CACpC,GAAG,eAAe,KAAKX,KAAK,WAAW,EAAE,GAAG,IAAI,KAAK,KACrD,GAAG,KAAKY,uBAAuB,KAAK,MAAM,CAC5C,CAAC;EACH,OAAO;GAML,GAAG,MAAM,KAAK,KAAK;GACnB,IAAI,GAAG,MAAM,UAAU,KAAKnB,qBAAqB;IAK/C,MAAM,YAAY,GAAG;IACrB,KAAUkB,gBAAgB,EAAE;IAC5B,MAAM;GACR;EACF;EACA,IAAI,QAAQ,UAAU;GAIpB,KAAUA,gBAAgB,EAAE;GAE5B,MAAM,EAAC,UAAS,MAAM,GAAG;GACzB,IAAI,UAAU,KAAKhC,SAEjB,GAAG,KAAK,KACN,IAAI,WAAW,2CAA2C,OAAO,CACnE;QACK;IAEL,MAAM,gBAAgB;IACtB,GAAQ,KAAK,SAAQ,OAAM,CACzB,EAAE;iBACK,KAAKqB,KAAK,kBAAkB,EAAE,OAAO,GAAG,EAAC,cAAa,CAAC,GAChE,CAAC;IACD,GAAG,KAAK,QAAQ;GAClB;GAEA,MAAM,GAAG,KAAK,KAAK;GAEnB,MAAM,EAAC,kBAAkB,QAAO;GAChC,KAAK;GACL,IAAI,kBACF,KAAKQ,kBAAkB,gBAAgB;GAGzC,IAAI,KACF,KAAKpB,aAAa;IAAC;IAAU;IAAQ,EAAC,UAAS;GAAC,CAAC;EAErD,OAAO,IAAI,QAAQ,YAAY;GAE7B,GAAG,KAAK,MAAM;GACd,MAAM,GAAG,KAAK,KAAK;GACnB,KAAK;EACP;EACA,OAAO;CACT;CAEA,MAAMmB,cAAc,MAA2B;EAC7C,MAAM,cAAc,KAAK;EACzB,IAAI,gBAAgB,GAClB;EAGF,MAAM,KAAK,KAAK9B,IAAI,YAAY,QAAQ,SAAS;EACjD,MAAM,EAAC,MAAM,SAAS,eAAc,eAAe;EACnD,MAAM,SAAS,IAAI,gBAAgB,IAAI;GACrC,MAAM;GACN;GACA;GACA,gBAAgB,KAAK;EACvB,CAAC;EAID,MAAM,eAAe,KAAKK,oBACxB,sBACA,WACF;EACA,OAAO,IAAI,YAAY;EAEvB,IAAI;EACJ,MAAM,qBAAqB,KAAKa,iBAAiB,UAAU;GACzD,MAAM;GACN,aAAa,KAAK,KAAK,EAAC,YAAY,QAAO,EAAE,EAAE;EACjD,CAAC;EACD,IAAI;GAKF,CAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAA8B;sCACP,KAAKK,KAAK,kBAAkB,EAAE;OAE9D;GACA,GAAG,OACD,qBAAqB,cAAc,IAAI,MAAM,WAAW,eACxC,YAAY,eAC9B;EACF,SAAS,GAAG;GACV,KAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,CAAC,CAAC;GAI7C,OAAO,MAAM;GACb,aAAkB,IAAI,EAAE,YAAY,CAAC,CAAC;GACtC,MAAM;EACR,UAAU;GACR,mBAAmB;EACrB;EAIA,QAAa,WACX,KAAK,KAAI,QACP,KAAKa,SACH,GAAG,YAAY,cAAc,IAAI,WAAW,EAAE,GAC9C,KACA,eACA,MACF,CACF,CACF,EAAE,cAAc;GACd,OAAO,QAAQ;GACf,aAAkB,IAAI,EAAE,YAAY,CAAC,CAAC;EACxC,CAAC;CACH;CAEA,MAAMA,SACJ,IACA,EAAC,YAAY,KAAK,QAClB,eACA,QACA;EACA,IAAI,kBAAkB;EACtB,IAAI,oBAAoB,KAAKlB,iBAAiB,UAAU;GACtD,MAAM;GACN,YAAY,IAAI;GAChB;EACF,CAAC;EACD,IAAI;GACF,GAAG,OAAO,kBAAkB;GAC5B,MAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,GAAG,OAAO,aAAa;IACvB,MAAM,QAAQ,KAAK,IAAI;IAIvB,IAAI,iBAAiB,IAAI,cAAc,KAAKR;IAC5C,IAAI,QAAQ;IACZ,IAAI;IAEJ,WAAW,MAAM,WAAW,EAA6B;sEACK,KAAKa,KAAK,WAAW,EAAE;gCAC7D,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,GAAI,GAAG;KACxC,kBAAkB;KAClB,mBAAmB,QAAQ;KAE3B,IAAI;MAOF,MAAM,QAAQ,YAAY,IAAI;MAC9B,MAAM;MACN,MAAM,UAAU,YAAY,IAAI,IAAI;MACpC,IAAI,mBACF,GAAG,UAAU,MAAM,SAAS,WAC1B,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,IAAI,GAAG,0CAChD;MAGF,KAAK,MAAM,SAAS,SAClB,IAAI,MAAM,cAAc,IAAI,WAG1B,iBAAiB;WACZ,IAAI,gBAAgB;OACzB,oBAAoB,IAAI,QACtB,6BAA6B,KAAK,CACpC;OACA;MACF,OAAO,IAAI,SAAS,UAClB,MAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,EACvF;WACK;OACL,GAAG,OACD,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,EAC7F;OACA,IAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,EACjF;OACA;MACF;KAEJ,UAAU;MAER,oBAAoB,KAAKL,iBAAiB,UAAU;OAClD,MAAM;OACN,YAAY,IAAI;OAChB;MACF,CAAC;KACH;IACF;IACA,kBAAkB;IAElB,IAAI,gBAAgB;KAClB,MAAM;KACN,GAAG,OACD,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,IAAI,IAAI,MACd,KACH;IACF,OAWE,GAAG,OACD,cAAc,IAAI,GAAG,gBAAgB,IAAI,UAAU,4CACjB,cAAc,wCAElD;IAMF,IAAS,YAAY;GACvB,CAAC;EACH,SAAS,KAAK;GACZ,GAAG,QAAQ,sCAAsC,IAAI,MAAM,GAAG;GAC9D,IAAI,eAAe,iBAAiB;IAClC,MAAM,kBAAkB,KAAKZ,KAAK,KAAKL,MAAM;IAC7C,KAAKW,SAAS,GAAG;GACnB;GACA,IAAI,KAAK,GAAG;EACd,UAAU;GACR,kBAAkB;EACpB;CACF;;;;;;;;;;;;CAaA,uBAAuB,KAA0B,QAAsB;EACrE,OAAO,UAAU,MAAM,EAAE,SAAQ,OAAM,KAAKyB,aAAa,KAAK,EAAE,CAAC;CACnE;CAEA,aAAa,KAA0B,IAAqC;EAC1E,QAAQ,GAAG,IAAX;GACE,KAAK,mBACH,OAAO,CAAC,KAAKC,yBAAyB,KAAK,GAAG,OAAO,GAAG,QAAQ,CAAC;GAEnE,KAAK,mBACH,OAAO,CACL,KAAKC,0BAA0B,KAAK,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CACtE;GAEF,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,GAAG,IAAI;KAAQ,OAAO,GAAG,IAAI;IAAI;IACtD,OAAO,CACL,GAAG,UAAU,KAAKhB,KAAK,eAAe,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,OAAO,IAAI,GAAG,EAAE;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,MAC3D;GACF;GAEA,KAAK,cAAc;IACjB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,eAAe,EAAE;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MACnD;GACF;GAEA,KAAK,iBAAiB;IACpB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,UAAU,KAAKA,KAAK,aAAa,EAAE,kBAAkB,GAAG,IAAI;mCACtC,OAAO,iBAAiB,MAAM,kBAAkB,GAAG,KAC9E;GACF;GAEA,KAAK,eAAe;IAClB,MAAM,EAAC,QAAQ,SAAQ,GAAG;IAC1B,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,GAAG,QAC7E;GACF;GAEA,KAAK,qBAAqB;IACxB,MAAM,EAAC,QAAQ,MAAM,UAAS,GAAG;IACjC,OAAO,CACL,GAAG,eAAe,KAAKA,KAAK,aAAa,EAAE;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAC/F;GACF;GAEA,SACE,YAAY,EAAE;EAClB;CACF;CAEA,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;EAAQ;EACtD,OAAO,GAAG;sBACQ,KAAKA,KAAK,eAAe,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEnC,IAAI,GAAG,EAAE;;CAEjC;CAEA,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;EAAQ;EAC/D,OAAO,GAAG;sBACQ,KAAKA,KAAK,aAAa,EAAE,GAAG,IAAI,GAAG,EAAE;;0BAEjC,IAAI,GAAG,EAAE;;CAEjC;;;;;CAMA,MAAM,eAAe;EACnB,IAAI,KAAKE,UAAU;GACjB,MAAM,EAAC,SAAS,YAAW,SAAS;GACpC,KAAKZ,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;GACtC,MAAM;EACR;CACF;;;;;;CAOA,MAAM,OAAO;EACX,IAAI,KAAKY,UAAU;GACjB,KAAKP,iBAAiB,KAAK;GAC3B,KAAKlB,IAAI,OAAO,YAAY,KAAKa,OAAO,KAAK,EAAE,mBAAmB;GAClE,KAAK,MAAM;GACX,KAAKA,OAAO,QAAQ,MAAM;EAC5B;EACA,IACG,MAAM,UAAU,KAAKc,UAAU,KAAKV,eAAe,MAAO,aAE3D,MAAM,IAAI,WACR,kCAAkC,KAAKA,gBAAgB,GACzD;CAEJ;AACF;AAEA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;;;;;;CAMA;CAEA,YACE,IACA,IACA,gBACA,WACA,eACA;EACA,KAAKjB,MAAM;EACX,KAAKwC,MAAM;EACX,KAAK,iBAAiB;EACtB,KAAK,eAAe;EACpB,KAAK,gBAAgB;CACvB;CAEA,YAAY;CAEZ,MAAM,UAAU;EACd,IAAI,KAAKC,WACP;EAEF,KAAKA,YAAY;EACjB,KAAKD,IAAI,QAAQ;EACjB,MAAM,KAAKA,IACR,KAAK,EACL,OAAM,MAAK,KAAKxC,IAAI,OAAO,iCAAiC,CAAC,CAAC;EACjE,KAAKA,IAAI,OAAO,0BAA0B,KAAK,cAAc;CAC/D;AACF;AAEA,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB,IAAgB;EAC1D,KAAKA,MAAM,GAAG,YAAY,aAAa,cAAc;EACrD,KAAKC,SAAS;EACd,KAAKK,MAAM;CACb;CAGA,KAAK,OAAe;EAClB,OAAO,KAAKA,IAAI,GAAG,UAAU,KAAKL,MAAM,EAAE,GAAG,OAAO;CACtD;CAkBA,MAAM,QACJ,eAC2C;EAC3C,MAAM,KAAK,IAAI,gBAAgB,KAAKD,KAAK,EAAC,MAAM,eAAmB,CAAC,EAAE,IACpE,KAAKM,GACP;EACA,MAAM,CAAC,SAAS,MAAM,GAAG,iBACvB,QAAO,GAA8B;oCACP,KAAKiB,KAAK,kBAAkB,GAC5D;EAGA,MAAM,gBAAgB,OAAO;EAC7B,IACE,kBAAkB,KAAA,MACjB,kBAAkB,KAAA,KAAa,gBAAgB,gBAChD;GACA,KAAKvB,IAAI,OACP,aAAa,iBAAiB,kBAAkB,kCAAkC,eACpF;GACA,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,IAAI,kBAAkB,KAAA,GAAW;GAC/B,KAAKA,IAAI,OAAO,oDAAoD;GACpE,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,MAAM,MAAM,MAAM,GAAG,iBACnB,QAAO,GAA0B;8BACT,KAAKuB,KAAK,WAAW,EAAE;;;KAIjD;EACA,IAAI,IAAI,WAAW,GAAG;GACpB,KAAKvB,IAAI,OAAO,4CAA4C;GAC5D,GAAG,QAAQ;GACX,MAAM,GAAG,KAAK;GACd,OAAO;EACT;EACA,MAAM,CAAC,EAAC,eAAc;EACtB,MAAM,CAAC,EAAC,oBAAmB,MAAM,GAAG,iBAClC,QAAO,GAA+B;uCACL,KAAKuB,KAAK,mBAAmB,EAAE;OAElE;EACA,KAAKvB,IAAI,OACP,oBAAoB,UAAU,kCAAkC,gBAClE;EACA,OAAO,IAAI,UACT,KAAKA,KACL,IACA,gBACA,WACA,aACF;CACF;AACF;;;;;;;;;;;;;;;;;;AA0BA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA,+BAAwB,IAAI,IAAmB;CAE/C;CAEA,YACE,IACA,oBACA,WACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,yBAAyB;EAChE,KAAK0C,sBAAsB;EAC3B,KAAKC,aAAa;CACpB;CAEA,QAAQ;EACN,cAAc,KAAKE,MAAM;EACzB,KAAKA,SAAS,kBACN,KAAK,kCAAkB,IAAI,KAAK,CAAC,GACvC,KAAKH,mBACP,EAAE,MAAM;CACV;CAGA,kBAAkB,KAAW;EAC3B,KAAK,MAAM,CAAC,MAAM,eAAe,KAAKE,aAAa,QAAQ,GACzD,IAAI,IAAI,QAAQ,IAAI,WAAW,QAAQ,KAAK,KAAKF,qBAAqB;GACpE,KAAK1C,IAAI,QACP,kCAAkC,KAAK0C,oBAAoB,SAC3D;IAAC;IAAM;GAAU,CACnB;GACA,KAAKC,2BACH,IAAI,MACF,oCAAoC,KAAKD,oBAAoB,MAAM,WAAW,UAAU,IAAI,GAC9F,CACF;GACA,KAAK,KAAK;GACV;EACF;CAEJ;;;;;;CAOA,UAAU,MAAe,sBAAM,IAAI,KAAK,GAAe;EACrD,KAAKE,aAAa,IAAI,MAAM,GAAG;EAC/B,aAAa;GACX,KAAKA,aAAa,OAAO,IAAI;EAC/B;CACF;CAEA,OAAO;EACL,cAAc,KAAKC,MAAM;EACzB,KAAKD,aAAa,MAAM;CAC1B;AACF"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { LogContext } from '@rocicorp/logger';
|
|
2
|
+
/**
|
|
3
|
+
* Whether the imported postgres.js has Zero's patch
|
|
4
|
+
* (`packages/zero/patches/postgres@3.4.7.patch`), which marks itself with
|
|
5
|
+
* `Postgres.zeroPatch`. pnpm applies the patch in this repo and in the Docker
|
|
6
|
+
* image, but not when `@rocicorp/zero` is installed from npm.
|
|
7
|
+
*
|
|
8
|
+
* The patch includes the `CopyDone` fix for
|
|
9
|
+
* https://github.com/porsager/postgres/issues/499
|
|
10
|
+
* (https://github.com/porsager/postgres/pull/1016). Without it, a connection
|
|
11
|
+
* can stop responding after a COPY, on any Node version: if the last
|
|
12
|
+
* `CopyData` fills the stream's buffer, the socket is paused, and it is never
|
|
13
|
+
* resumed once the stream has ended. Node version and OS only change how often
|
|
14
|
+
* this happens. Initial sync and backfill only wait for their COPY connections
|
|
15
|
+
* to close when this returns `true`.
|
|
16
|
+
*
|
|
17
|
+
* A later version of the patch can raise the number to mark new fixes.
|
|
18
|
+
*/
|
|
19
|
+
export declare function hasPostgresPatch(): boolean;
|
|
20
|
+
export declare function warnIfPostgresUnpatched(lc: LogContext): void;
|
|
21
|
+
//# sourceMappingURL=pg-patch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pg-patch.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/pg-patch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,gBAAgB,IAAI,OAAO,CAE1C;AAED,wBAAgB,uBAAuB,CAAC,EAAE,EAAE,UAAU,GAAG,IAAI,CAU5D"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import postgres from "postgres";
|
|
2
|
+
//#region ../zero-cache/src/types/pg-patch.ts
|
|
3
|
+
/**
|
|
4
|
+
* Whether the imported postgres.js has Zero's patch
|
|
5
|
+
* (`packages/zero/patches/postgres@3.4.7.patch`), which marks itself with
|
|
6
|
+
* `Postgres.zeroPatch`. pnpm applies the patch in this repo and in the Docker
|
|
7
|
+
* image, but not when `@rocicorp/zero` is installed from npm.
|
|
8
|
+
*
|
|
9
|
+
* The patch includes the `CopyDone` fix for
|
|
10
|
+
* https://github.com/porsager/postgres/issues/499
|
|
11
|
+
* (https://github.com/porsager/postgres/pull/1016). Without it, a connection
|
|
12
|
+
* can stop responding after a COPY, on any Node version: if the last
|
|
13
|
+
* `CopyData` fills the stream's buffer, the socket is paused, and it is never
|
|
14
|
+
* resumed once the stream has ended. Node version and OS only change how often
|
|
15
|
+
* this happens. Initial sync and backfill only wait for their COPY connections
|
|
16
|
+
* to close when this returns `true`.
|
|
17
|
+
*
|
|
18
|
+
* A later version of the patch can raise the number to mark new fixes.
|
|
19
|
+
*/
|
|
20
|
+
function hasPostgresPatch() {
|
|
21
|
+
return postgres.zeroPatch !== void 0;
|
|
22
|
+
}
|
|
23
|
+
function warnIfPostgresUnpatched(lc) {
|
|
24
|
+
if (!hasPostgresPatch()) lc.warn?.("postgres.js does not have Zero's patch for a bug that can leave a Postgres connection unresponsive after a COPY, on any Node version (https://github.com/porsager/postgres/pull/1016). Initial sync and backfill will not wait for their COPY connections to close, as in zero-cache 1.10. The rocicorp/zero Docker image includes the patch.");
|
|
25
|
+
}
|
|
26
|
+
//#endregion
|
|
27
|
+
export { hasPostgresPatch, warnIfPostgresUnpatched };
|
|
28
|
+
|
|
29
|
+
//# sourceMappingURL=pg-patch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pg-patch.js","names":[],"sources":["../../../../../zero-cache/src/types/pg-patch.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\n\n/**\n * Whether the imported postgres.js has Zero's patch\n * (`packages/zero/patches/postgres@3.4.7.patch`), which marks itself with\n * `Postgres.zeroPatch`. pnpm applies the patch in this repo and in the Docker\n * image, but not when `@rocicorp/zero` is installed from npm.\n *\n * The patch includes the `CopyDone` fix for\n * https://github.com/porsager/postgres/issues/499\n * (https://github.com/porsager/postgres/pull/1016). Without it, a connection\n * can stop responding after a COPY, on any Node version: if the last\n * `CopyData` fills the stream's buffer, the socket is paused, and it is never\n * resumed once the stream has ended. Node version and OS only change how often\n * this happens. Initial sync and backfill only wait for their COPY connections\n * to close when this returns `true`.\n *\n * A later version of the patch can raise the number to mark new fixes.\n */\nexport function hasPostgresPatch(): boolean {\n return (postgres as {zeroPatch?: number}).zeroPatch !== undefined;\n}\n\nexport function warnIfPostgresUnpatched(lc: LogContext): void {\n if (!hasPostgresPatch()) {\n lc.warn?.(\n `postgres.js does not have Zero's patch for a bug that can leave a ` +\n `Postgres connection unresponsive after a COPY, on any Node version ` +\n `(https://github.com/porsager/postgres/pull/1016). Initial sync and ` +\n `backfill will not wait for their COPY connections to close, as in ` +\n `zero-cache 1.10. The rocicorp/zero Docker image includes the patch.`,\n );\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,mBAA4B;CAC1C,OAAQ,SAAkC,cAAc,KAAA;AAC1D;AAEA,SAAgB,wBAAwB,IAAsB;CAC5D,IAAI,CAAC,iBAAiB,GACpB,GAAG,OACD,+UAKF;AAEJ"}
|
|
@@ -31,7 +31,7 @@ var ConnectionManager = class ConnectionManager extends Subscribable {
|
|
|
31
31
|
#timeoutCheckIntervalMs;
|
|
32
32
|
/**
|
|
33
33
|
* `Date.now()` at the last timeout-interval tick, used to notice that the
|
|
34
|
-
* JavaScript event loop was not running. See {@
|
|
34
|
+
* JavaScript event loop was not running. See {@link #creditFrozenTime}.
|
|
35
35
|
*/
|
|
36
36
|
#lastTimeoutCheckAt;
|
|
37
37
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connection-manager.js","names":["#disconnectTimeout","#timeoutCheckIntervalMs","#state","#lastTimeoutCheckAt","#nextStatePromise","#connectRequestResolver","#consumeConnectRequest","#connectingStartedAt","#maybeStartTimeoutInterval","#publishStateAndGetPromise","#maybeStopTimeoutInterval","#publishState","#resolveNextStateWaiters","#stateChangeResolver","#timeoutInterval","#creditFrozenTime","#checkTimeout"],"sources":["../../../../../zero-client/src/client/connection-manager.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {Subscribable} from '../../../shared/src/subscribable.ts';\nimport {ClientErrorKind} from './client-error-kind.ts';\nimport {ConnectionStatus} from './connection-status.ts';\nimport {\n ClientError,\n isClientError,\n type AuthError,\n type ClosedError,\n type DisconnectedReason,\n type ZeroError,\n} from './error.ts';\n\nconst DEFAULT_TIMEOUT_CHECK_INTERVAL_MS = 1_000;\n\nexport type ConnectionManagerState =\n | {\n name: ConnectionStatus.Initializing;\n }\n | {\n name: ConnectionStatus.Disconnected;\n reason: DisconnectedReason;\n }\n | {\n name: ConnectionStatus.Connecting;\n attempt: number;\n disconnectAt: number;\n reason?: ZeroError | undefined;\n }\n | {\n name: ConnectionStatus.Connected;\n }\n | {\n name: ConnectionStatus.NeedsAuth;\n reason: AuthError;\n }\n | {\n name: ConnectionStatus.Error;\n reason: ZeroError;\n }\n | {\n name: ConnectionStatus.Closed;\n reason: ClosedError;\n };\n\nexport type ConnectionManagerOptions = {\n /**\n * The amount of milliseconds we allow for continuous connecting attempts before\n * transitioning to disconnected state.\n */\n disconnectTimeout: number;\n /**\n * How frequently we check whether the connecting timeout has elapsed.\n * Defaults to 1 second.\n */\n timeoutCheckIntervalMs?: number | undefined;\n};\n\nconst TERMINAL_STATES = [\n ConnectionStatus.NeedsAuth,\n ConnectionStatus.Error,\n] as const satisfies ConnectionStatus[];\n\ntype TerminalConnectionStatus = (typeof TERMINAL_STATES)[number];\ntype TerminalConnectionManagerState = Extract<\n ConnectionManagerState,\n {name: TerminalConnectionStatus}\n>;\n\nexport class ConnectionManager extends Subscribable<ConnectionManagerState> {\n #state: ConnectionManagerState;\n #connectRequestResolver: Resolver<void> | undefined = resolver();\n\n /**\n * The timestamp when we first started trying to connect.\n * This is used to track the retry window.\n * Reset to undefined when we successfully connect or when we transition to disconnected.\n */\n #connectingStartedAt: number | undefined;\n\n /**\n * The amount of milliseconds we allow for continuous connecting attempts before\n * transitioning to disconnected state.\n */\n #disconnectTimeout: number;\n\n /**\n * Handle for the timeout interval that periodically checks whether we've\n * exceeded the allowed connecting window.\n */\n #timeoutInterval: ReturnType<typeof setInterval> | undefined;\n\n /**\n * Interval duration for checking whether the connecting timeout has elapsed.\n */\n #timeoutCheckIntervalMs: number;\n\n /**\n * `Date.now()` at the last timeout-interval tick, used to notice that the\n * JavaScript event loop was not running. See {@linkcode #creditFrozenTime}.\n */\n #lastTimeoutCheckAt: number;\n\n /**\n * Resolver used to signal waiting callers when the state changes.\n */\n #stateChangeResolver: Resolver<ConnectionManagerState> = resolver();\n\n constructor(options: ConnectionManagerOptions) {\n super();\n\n this.#disconnectTimeout = options.disconnectTimeout;\n this.#timeoutCheckIntervalMs =\n options.timeoutCheckIntervalMs ?? DEFAULT_TIMEOUT_CHECK_INTERVAL_MS;\n // The connecting window, and the timer that enforces it, start in\n // initialized(), once the local store is loaded.\n this.#state = {name: ConnectionStatus.Initializing};\n this.#lastTimeoutCheckAt = Date.now();\n }\n\n get state(): ConnectionManagerState {\n return this.#state;\n }\n\n /**\n * Returns true if the current state is equal to the given status.\n */\n is(status: ConnectionStatus): boolean {\n return this.#state.name === status;\n }\n\n /**\n * Returns true if the current state is a terminal state\n * that can be recovered from by calling connect().\n */\n isInTerminalState(): boolean {\n return ConnectionManager.isTerminalState(this.#state);\n }\n\n /**\n * Returns true if the given status is a terminal state\n * that can be recovered from by calling connect().\n */\n static isTerminalState(\n state: ConnectionManagerState,\n ): state is TerminalConnectionManagerState {\n return (TERMINAL_STATES as readonly ConnectionStatus[]).includes(\n state.name,\n );\n }\n\n /**\n * Returns true if the run loop should continue.\n * The run loop continues in all states except closed.\n * In needs-auth and error states, the run loop pauses and waits for connect() to be called.\n */\n shouldContinueRunLoop(): boolean {\n return this.#state.name !== ConnectionStatus.Closed;\n }\n\n /**\n * Waits for the next state change.\n * @returns A promise that resolves when the next state change occurs.\n */\n waitForStateChange(): Promise<ConnectionManagerState> {\n return this.#nextStatePromise();\n }\n\n requestConnect(): void {\n if (this.#connectRequestResolver === undefined) {\n return;\n }\n this.#connectRequestResolver.resolve();\n this.#connectRequestResolver = undefined;\n }\n\n waitForConnectRequest(): Promise<void> {\n return this.#connectRequestResolver === undefined\n ? Promise.resolve()\n : this.#connectRequestResolver.promise;\n }\n\n #consumeConnectRequest(): boolean {\n if (this.#connectRequestResolver !== undefined) {\n return false;\n }\n this.#connectRequestResolver = resolver();\n return true;\n }\n\n /**\n * Consume a pending connect request and resume connecting.\n *\n * @returns true if a pending request was handled.\n */\n resumeFromConnectRequest(): boolean {\n if (!this.isInTerminalState()) {\n return false;\n }\n if (!this.#consumeConnectRequest()) {\n return false;\n }\n this.connecting();\n return true;\n }\n\n /**\n * Transition from initializing to connecting, once the local store is\n * loaded.\n *\n * This is where the connecting window starts. Loading the local store can\n * take tens of seconds on a slow device with a large replica, and none of\n * that says anything about whether the server is reachable, so it is not\n * charged against the disconnect timeout. No connect attempt has been made\n * yet, so `attempt` is 0.\n *\n * A no-op in any other state: something else, like `close()`, has already\n * moved the state on.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n initialized(): {nextStatePromise: Promise<ConnectionManagerState>} {\n if (this.#state.name !== ConnectionStatus.Initializing) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n const now = Date.now();\n this.#connectingStartedAt = now;\n this.#state = {\n name: ConnectionStatus.Connecting,\n attempt: 0,\n disconnectAt: now + this.#disconnectTimeout,\n };\n // Start the interval before publishing: a subscriber may close the manager\n // synchronously, and closed() can only stop an interval that exists.\n this.#maybeStartTimeoutInterval();\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to connecting state.\n *\n * This starts the timeout timer, but if we've entered disconnected state,\n * we stay there and continue retrying.\n *\n * If we're disconnected due to a hidden tab, allow a transition back to\n * connecting when visibility returns.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n connecting(reason?: ZeroError): {\n nextStatePromise: Promise<ConnectionManagerState>;\n } {\n // cannot transition from closed to any other status\n if (this.#state.name === ConnectionStatus.Closed) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n const isHiddenDisconnect =\n this.#state.name === ConnectionStatus.Disconnected &&\n this.#state.reason.kind === ClientErrorKind.Hidden;\n\n // we cannot intentionally transition from disconnected to connecting\n // unless the disconnection was due to a hidden tab\n if (\n this.#state.name === ConnectionStatus.Disconnected &&\n !isHiddenDisconnect\n ) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n const now = Date.now();\n\n if (isHiddenDisconnect) {\n // Reset the retry window after a hidden-tab disconnect.\n this.#connectingStartedAt = undefined;\n }\n\n // If we're already connecting, increment the attempt counter\n if (this.#state.name === ConnectionStatus.Connecting) {\n this.#state = {\n ...this.#state,\n attempt: this.#state.attempt + 1,\n reason,\n };\n // See initialized() for why the interval starts before publishing.\n this.#maybeStartTimeoutInterval();\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n // Starting a new connecting session\n // If #connectingStartedAt is undefined, this is a fresh start - set it to now\n // If it's already set, we're retrying within the same retry window, so keep it\n if (this.#connectingStartedAt === undefined) {\n this.#connectingStartedAt = now;\n }\n\n const disconnectAt = this.#connectingStartedAt + this.#disconnectTimeout;\n\n this.#state = {\n name: ConnectionStatus.Connecting,\n attempt: 1,\n disconnectAt,\n reason,\n };\n // Start the interval before publishing: a subscriber may close the manager\n // synchronously, and closed() can only stop an interval that exists.\n this.#maybeStartTimeoutInterval();\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to connected state.\n * This resets the connecting timeout timer.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n connected(): {nextStatePromise: Promise<ConnectionManagerState>} {\n // cannot transition from closed to any other status\n if (this.#state.name === ConnectionStatus.Closed) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Already connected, no-op\n if (this.#state.name === ConnectionStatus.Connected) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Reset the timeout timer on successful connection\n this.#connectingStartedAt = undefined;\n this.#maybeStopTimeoutInterval();\n\n this.#state = {\n name: ConnectionStatus.Connected,\n };\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to disconnected state.\n * This is called when the timeout expires.\n * The run loop will continue trying to reconnect.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n disconnected(reason: DisconnectedReason): {\n nextStatePromise: Promise<ConnectionManagerState>;\n } {\n // cannot transition from closed to any other status\n if (this.#state.name === ConnectionStatus.Closed) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Already disconnected, no-op\n if (this.#state.name === ConnectionStatus.Disconnected) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // When transitioning from connected to disconnected, we've lost a connection\n // we previously had. Clear the timeout timer so we can start a fresh timeout window.\n if (this.#state.name === ConnectionStatus.Connected) {\n this.#connectingStartedAt = undefined;\n }\n // When transitioning from connecting to disconnected (e.g., due to timeout),\n // we keep the start time to maintain the context that we've been trying for a while.\n\n this.#maybeStopTimeoutInterval();\n\n this.#state = {\n name: ConnectionStatus.Disconnected,\n reason,\n };\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to needs-auth state.\n * This pauses the run loop until connect() is called with new credentials.\n * Resets the retry window and attempt counter.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n needsAuth(reason: AuthError): {\n nextStatePromise: Promise<ConnectionManagerState>;\n } {\n // cannot transition from closed to any other status\n if (this.#state.name === ConnectionStatus.Closed) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Already in needs-auth state, no-op\n if (this.#state.name === ConnectionStatus.NeedsAuth) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Reset the timeout timer and connecting start time\n this.#connectingStartedAt = undefined;\n this.#maybeStopTimeoutInterval();\n\n this.#state = {\n name: ConnectionStatus.NeedsAuth,\n reason,\n };\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to error state.\n * This pauses the run loop until connect() is called.\n * Resets the retry window and attempt counter.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n error(reason: ZeroError): {\n nextStatePromise: Promise<ConnectionManagerState>;\n } {\n // cannot transition from closed to any other status\n if (this.#state.name === ConnectionStatus.Closed) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Already in error state, no-op\n if (this.#state.name === ConnectionStatus.Error) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Reset the timeout timer and connecting start time\n this.#connectingStartedAt = undefined;\n this.#maybeStopTimeoutInterval();\n\n this.#state = {\n name: ConnectionStatus.Error,\n reason,\n };\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to closed state.\n * This is terminal - no further transitions are allowed.\n */\n closed() {\n // Already closed, no-op\n if (this.#state.name === ConnectionStatus.Closed) {\n return;\n }\n\n this.#connectingStartedAt = undefined;\n this.#maybeStopTimeoutInterval();\n\n this.#state = {\n name: ConnectionStatus.Closed,\n reason: new ClientError({\n kind: ClientErrorKind.ClientClosed,\n message: 'Zero was explicitly closed by calling zero.close()',\n }),\n };\n this.#publishState();\n this.cleanup();\n return;\n }\n\n override cleanup = (): void => {\n this._listeners.clear();\n this.#resolveNextStateWaiters();\n };\n\n #resolveNextStateWaiters(): void {\n this.#stateChangeResolver.resolve(this.#state);\n this.#stateChangeResolver = resolver();\n }\n\n #publishState(): void {\n this.notify(this.#state);\n this.#resolveNextStateWaiters();\n }\n\n #nextStatePromise(): Promise<ConnectionManagerState> {\n return this.#stateChangeResolver.promise;\n }\n\n #publishStateAndGetPromise(): Promise<ConnectionManagerState> {\n this.#publishState();\n return this.#nextStatePromise();\n }\n\n /**\n * Pushes the disconnect deadline forward by however long the event loop was\n * not running.\n *\n * `disconnectAt` is an absolute wall-clock instant, but what it is meant to\n * bound is time spent *actually retrying*. Whenever the loop is frozen --- a\n * suspended React Native app, a sleeping laptop, a throttled background tab\n * --- wall-clock keeps advancing while no reconnect attempt can run, so\n * without this the first tick after resume reports `Offline` having given the\n * client zero live seconds to connect.\n *\n * This interval is its own detector: a tick that lands late by more than a\n * full period means we were not running in between. A merely busy JS thread\n * can also delay a tick, and we cannot tell the two apart from lateness\n * alone, but the costs are asymmetric --- crediting a busy period back just\n * grants a bit more time to connect, whereas failing to credit a real freeze\n * produces a user-visible bogus disconnect --- so we credit.\n *\n * The whole gap is credited, not `elapsed` minus a period. An overdue\n * `setInterval` callback runs as soon as the loop resumes rather than waiting\n * out another period, so `elapsed` is already about the frozen duration;\n * netting off a period would charge up to a full period of frozen time\n * against the deadline. Freezing just after a tick would then leave the\n * deadline exactly at `now` on resume and disconnect immediately, which is\n * the very thing this is here to prevent.\n */\n #creditFrozenTime(): void {\n const now = Date.now();\n const elapsed = now - this.#lastTimeoutCheckAt;\n this.#lastTimeoutCheckAt = now;\n\n // Ordinary scheduling jitter, not a freeze.\n if (elapsed <= 2 * this.#timeoutCheckIntervalMs) {\n return;\n }\n const frozenMs = elapsed;\n\n // Advance the window's origin so that a later `connecting()` that starts a\n // fresh session recomputes a deadline that also excludes the frozen time.\n if (this.#connectingStartedAt !== undefined) {\n this.#connectingStartedAt += frozenMs;\n }\n\n if (this.#state.name === ConnectionStatus.Connecting) {\n // Deliberately not published: the status has not changed, and waking the\n // run loop's `waitForStateChange` racers here would interrupt an\n // in-flight connect attempt for no reason. Readers of `state` still see\n // the corrected deadline.\n this.#state = {\n ...this.#state,\n disconnectAt: this.#state.disconnectAt + frozenMs,\n };\n }\n }\n\n /**\n * Check if we should transition from connecting to disconnected due to timeout.\n * Returns true if the transition happened.\n */\n #checkTimeout(): boolean {\n if (this.#state.name !== ConnectionStatus.Connecting) {\n return false;\n }\n\n const now = Date.now();\n if (now >= this.#state.disconnectAt) {\n this.disconnected(\n new ClientError({\n kind: ClientErrorKind.Offline,\n message: `Zero was unable to connect for ${Math.floor(this.#disconnectTimeout / 1_000)} seconds and was disconnected`,\n }),\n );\n return true;\n }\n\n return false;\n }\n\n #maybeStartTimeoutInterval(): void {\n if (this.#timeoutInterval !== undefined) {\n return;\n }\n this.#lastTimeoutCheckAt = Date.now();\n this.#timeoutInterval = setInterval(() => {\n this.#creditFrozenTime();\n this.#checkTimeout();\n }, this.#timeoutCheckIntervalMs);\n }\n\n #maybeStopTimeoutInterval(): void {\n if (this.#timeoutInterval === undefined) {\n return;\n }\n clearInterval(this.#timeoutInterval);\n this.#timeoutInterval = undefined;\n }\n}\n\n/**\n * Used to trigger the catch block when a terminal state is reached.\n *\n * @param state - The current connection state.\n */\nexport const throwIfConnectionError = (state: ConnectionManagerState) => {\n if (\n ConnectionManager.isTerminalState(state) ||\n state.name === ConnectionStatus.Closed ||\n ((state.name === ConnectionStatus.Connecting ||\n state.name === ConnectionStatus.Disconnected) &&\n state.reason)\n ) {\n if (\n isClientError(state.reason) &&\n (state.reason.kind === ClientErrorKind.ConnectTimeout ||\n state.reason.kind === ClientErrorKind.AbruptClose ||\n state.reason.kind === ClientErrorKind.CleanClose ||\n state.reason.kind === ClientErrorKind.Hidden)\n ) {\n return;\n }\n throw state.reason;\n }\n};\n"],"mappings":";;;;;;AAaA,IAAM,oCAAoC;AA6C1C,IAAM,kBAAkB,CACtB,WACA,KACF;AAQA,IAAa,oBAAb,MAAa,0BAA0B,aAAqC;CAC1E;CACA,0BAAsD,SAAS;;;;;;CAO/D;;;;;CAMA;;;;;CAMA;;;;CAKA;;;;;CAMA;;;;CAKA,uBAAyD,SAAS;CAElE,YAAY,SAAmC;EAC7C,MAAM;EAEN,KAAKA,qBAAqB,QAAQ;EAClC,KAAKC,0BACH,QAAQ,0BAA0B;EAGpC,KAAKC,SAAS,EAAC,MAAM,aAA6B;EAClD,KAAKC,sBAAsB,KAAK,IAAI;CACtC;CAEA,IAAI,QAAgC;EAClC,OAAO,KAAKD;CACd;;;;CAKA,GAAG,QAAmC;EACpC,OAAO,KAAKA,OAAO,SAAS;CAC9B;;;;;CAMA,oBAA6B;EAC3B,OAAO,kBAAkB,gBAAgB,KAAKA,MAAM;CACtD;;;;;CAMA,OAAO,gBACL,OACyC;EACzC,OAAQ,gBAAgD,SACtD,MAAM,IACR;CACF;;;;;;CAOA,wBAAiC;EAC/B,OAAO,KAAKA,OAAO,SAAS;CAC9B;;;;;CAMA,qBAAsD;EACpD,OAAO,KAAKE,kBAAkB;CAChC;CAEA,iBAAuB;EACrB,IAAI,KAAKC,4BAA4B,KAAA,GACnC;EAEF,KAAKA,wBAAwB,QAAQ;EACrC,KAAKA,0BAA0B,KAAA;CACjC;CAEA,wBAAuC;EACrC,OAAO,KAAKA,4BAA4B,KAAA,IACpC,QAAQ,QAAQ,IAChB,KAAKA,wBAAwB;CACnC;CAEA,yBAAkC;EAChC,IAAI,KAAKA,4BAA4B,KAAA,GACnC,OAAO;EAET,KAAKA,0BAA0B,SAAS;EACxC,OAAO;CACT;;;;;;CAOA,2BAAoC;EAClC,IAAI,CAAC,KAAK,kBAAkB,GAC1B,OAAO;EAET,IAAI,CAAC,KAAKC,uBAAuB,GAC/B,OAAO;EAET,KAAK,WAAW;EAChB,OAAO;CACT;;;;;;;;;;;;;;;;CAiBA,cAAmE;EACjE,IAAI,KAAKJ,OAAO,SAAS,gBACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAGpD,MAAM,MAAM,KAAK,IAAI;EACrB,KAAKG,uBAAuB;EAC5B,KAAKL,SAAS;GACZ,MAAM;GACN,SAAS;GACT,cAAc,MAAM,KAAKF;EAC3B;EAGA,KAAKQ,2BAA2B;EAEhC,OAAO,EAAC,kBADiB,KAAKC,2BACtB,EAAgB;CAC1B;;;;;;;;;;;;CAaA,WAAW,QAET;EAEA,IAAI,KAAKP,OAAO,SAAS,UACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAGpD,MAAM,qBACJ,KAAKF,OAAO,SAAS,kBACrB,KAAKA,OAAO,OAAO,SAAS;EAI9B,IACE,KAAKA,OAAO,SAAS,kBACrB,CAAC,oBAED,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAGpD,MAAM,MAAM,KAAK,IAAI;EAErB,IAAI,oBAEF,KAAKG,uBAAuB,KAAA;EAI9B,IAAI,KAAKL,OAAO,SAAS,cAA6B;GACpD,KAAKA,SAAS;IACZ,GAAG,KAAKA;IACR,SAAS,KAAKA,OAAO,UAAU;IAC/B;GACF;GAEA,KAAKM,2BAA2B;GAEhC,OAAO,EAAC,kBADiB,KAAKC,2BACtB,EAAgB;EAC1B;EAKA,IAAI,KAAKF,yBAAyB,KAAA,GAChC,KAAKA,uBAAuB;EAG9B,MAAM,eAAe,KAAKA,uBAAuB,KAAKP;EAEtD,KAAKE,SAAS;GACZ,MAAM;GACN,SAAS;GACT;GACA;EACF;EAGA,KAAKM,2BAA2B;EAEhC,OAAO,EAAC,kBADiB,KAAKC,2BACtB,EAAgB;CAC1B;;;;;;;CAQA,YAAiE;EAE/D,IAAI,KAAKP,OAAO,SAAS,UACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,IAAI,KAAKF,OAAO,SAAS,aACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,KAAKG,uBAAuB,KAAA;EAC5B,KAAKG,0BAA0B;EAE/B,KAAKR,SAAS,EACZ,MAAM,UACR;EAEA,OAAO,EAAC,kBADiB,KAAKO,2BACtB,EAAgB;CAC1B;;;;;;;;CASA,aAAa,QAEX;EAEA,IAAI,KAAKP,OAAO,SAAS,UACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,IAAI,KAAKF,OAAO,SAAS,gBACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAKpD,IAAI,KAAKF,OAAO,SAAS,aACvB,KAAKK,uBAAuB,KAAA;EAK9B,KAAKG,0BAA0B;EAE/B,KAAKR,SAAS;GACZ,MAAM;GACN;EACF;EAEA,OAAO,EAAC,kBADiB,KAAKO,2BACtB,EAAgB;CAC1B;;;;;;;;CASA,UAAU,QAER;EAEA,IAAI,KAAKP,OAAO,SAAS,UACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,IAAI,KAAKF,OAAO,SAAS,cACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,KAAKG,uBAAuB,KAAA;EAC5B,KAAKG,0BAA0B;EAE/B,KAAKR,SAAS;GACZ,MAAM;GACN;EACF;EAEA,OAAO,EAAC,kBADiB,KAAKO,2BACtB,EAAgB;CAC1B;;;;;;;;CASA,MAAM,QAEJ;EAEA,IAAI,KAAKP,OAAO,SAAS,UACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,IAAI,KAAKF,OAAO,SAAS,SACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,KAAKG,uBAAuB,KAAA;EAC5B,KAAKG,0BAA0B;EAE/B,KAAKR,SAAS;GACZ,MAAM;GACN;EACF;EAEA,OAAO,EAAC,kBADiB,KAAKO,2BACtB,EAAgB;CAC1B;;;;;CAMA,SAAS;EAEP,IAAI,KAAKP,OAAO,SAAS,UACvB;EAGF,KAAKK,uBAAuB,KAAA;EAC5B,KAAKG,0BAA0B;EAE/B,KAAKR,SAAS;GACZ,MAAM;GACN,QAAQ,IAAI,YAAY;IACtB,MAAM;IACN,SAAS;GACX,CAAC;EACH;EACA,KAAKS,cAAc;EACnB,KAAK,QAAQ;CAEf;CAEA,gBAA+B;EAC7B,KAAK,WAAW,MAAM;EACtB,KAAKC,yBAAyB;CAChC;CAEA,2BAAiC;EAC/B,KAAKC,qBAAqB,QAAQ,KAAKX,MAAM;EAC7C,KAAKW,uBAAuB,SAAS;CACvC;CAEA,gBAAsB;EACpB,KAAK,OAAO,KAAKX,MAAM;EACvB,KAAKU,yBAAyB;CAChC;CAEA,oBAAqD;EACnD,OAAO,KAAKC,qBAAqB;CACnC;CAEA,6BAA8D;EAC5D,KAAKF,cAAc;EACnB,OAAO,KAAKP,kBAAkB;CAChC;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4BA,oBAA0B;EACxB,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,UAAU,MAAM,KAAKD;EAC3B,KAAKA,sBAAsB;EAG3B,IAAI,WAAW,IAAI,KAAKF,yBACtB;EAEF,MAAM,WAAW;EAIjB,IAAI,KAAKM,yBAAyB,KAAA,GAChC,KAAKA,wBAAwB;EAG/B,IAAI,KAAKL,OAAO,SAAS,cAKvB,KAAKA,SAAS;GACZ,GAAG,KAAKA;GACR,cAAc,KAAKA,OAAO,eAAe;EAC3C;CAEJ;;;;;CAMA,gBAAyB;EACvB,IAAI,KAAKA,OAAO,SAAS,cACvB,OAAO;EAIT,IADY,KAAK,IACb,KAAO,KAAKA,OAAO,cAAc;GACnC,KAAK,aACH,IAAI,YAAY;IACd,MAAM;IACN,SAAS,kCAAkC,KAAK,MAAM,KAAKF,qBAAqB,GAAK,EAAE;GACzF,CAAC,CACH;GACA,OAAO;EACT;EAEA,OAAO;CACT;CAEA,6BAAmC;EACjC,IAAI,KAAKc,qBAAqB,KAAA,GAC5B;EAEF,KAAKX,sBAAsB,KAAK,IAAI;EACpC,KAAKW,mBAAmB,kBAAkB;GACxC,KAAKC,kBAAkB;GACvB,KAAKC,cAAc;EACrB,GAAG,KAAKf,uBAAuB;CACjC;CAEA,4BAAkC;EAChC,IAAI,KAAKa,qBAAqB,KAAA,GAC5B;EAEF,cAAc,KAAKA,gBAAgB;EACnC,KAAKA,mBAAmB,KAAA;CAC1B;AACF;;;;;;AAOA,IAAa,0BAA0B,UAAkC;CACvE,IACE,kBAAkB,gBAAgB,KAAK,KACvC,MAAM,SAAS,aACb,MAAM,SAAS,gBACf,MAAM,SAAS,mBACf,MAAM,QACR;EACA,IACE,cAAc,MAAM,MAAM,MACzB,MAAM,OAAO,SAAS,oBACrB,MAAM,OAAO,SAAS,iBACtB,MAAM,OAAO,SAAS,gBACtB,MAAM,OAAO,SAAS,WAExB;EAEF,MAAM,MAAM;CACd;AACF"}
|
|
1
|
+
{"version":3,"file":"connection-manager.js","names":["#disconnectTimeout","#timeoutCheckIntervalMs","#state","#lastTimeoutCheckAt","#nextStatePromise","#connectRequestResolver","#consumeConnectRequest","#connectingStartedAt","#maybeStartTimeoutInterval","#publishStateAndGetPromise","#maybeStopTimeoutInterval","#publishState","#resolveNextStateWaiters","#stateChangeResolver","#timeoutInterval","#creditFrozenTime","#checkTimeout"],"sources":["../../../../../zero-client/src/client/connection-manager.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {Subscribable} from '../../../shared/src/subscribable.ts';\nimport {ClientErrorKind} from './client-error-kind.ts';\nimport {ConnectionStatus} from './connection-status.ts';\nimport {\n ClientError,\n isClientError,\n type AuthError,\n type ClosedError,\n type DisconnectedReason,\n type ZeroError,\n} from './error.ts';\n\nconst DEFAULT_TIMEOUT_CHECK_INTERVAL_MS = 1_000;\n\nexport type ConnectionManagerState =\n | {\n name: ConnectionStatus.Initializing;\n }\n | {\n name: ConnectionStatus.Disconnected;\n reason: DisconnectedReason;\n }\n | {\n name: ConnectionStatus.Connecting;\n attempt: number;\n disconnectAt: number;\n reason?: ZeroError | undefined;\n }\n | {\n name: ConnectionStatus.Connected;\n }\n | {\n name: ConnectionStatus.NeedsAuth;\n reason: AuthError;\n }\n | {\n name: ConnectionStatus.Error;\n reason: ZeroError;\n }\n | {\n name: ConnectionStatus.Closed;\n reason: ClosedError;\n };\n\nexport type ConnectionManagerOptions = {\n /**\n * The amount of milliseconds we allow for continuous connecting attempts before\n * transitioning to disconnected state.\n */\n disconnectTimeout: number;\n /**\n * How frequently we check whether the connecting timeout has elapsed.\n * Defaults to 1 second.\n */\n timeoutCheckIntervalMs?: number | undefined;\n};\n\nconst TERMINAL_STATES = [\n ConnectionStatus.NeedsAuth,\n ConnectionStatus.Error,\n] as const satisfies ConnectionStatus[];\n\ntype TerminalConnectionStatus = (typeof TERMINAL_STATES)[number];\ntype TerminalConnectionManagerState = Extract<\n ConnectionManagerState,\n {name: TerminalConnectionStatus}\n>;\n\nexport class ConnectionManager extends Subscribable<ConnectionManagerState> {\n #state: ConnectionManagerState;\n #connectRequestResolver: Resolver<void> | undefined = resolver();\n\n /**\n * The timestamp when we first started trying to connect.\n * This is used to track the retry window.\n * Reset to undefined when we successfully connect or when we transition to disconnected.\n */\n #connectingStartedAt: number | undefined;\n\n /**\n * The amount of milliseconds we allow for continuous connecting attempts before\n * transitioning to disconnected state.\n */\n #disconnectTimeout: number;\n\n /**\n * Handle for the timeout interval that periodically checks whether we've\n * exceeded the allowed connecting window.\n */\n #timeoutInterval: ReturnType<typeof setInterval> | undefined;\n\n /**\n * Interval duration for checking whether the connecting timeout has elapsed.\n */\n #timeoutCheckIntervalMs: number;\n\n /**\n * `Date.now()` at the last timeout-interval tick, used to notice that the\n * JavaScript event loop was not running. See {@link #creditFrozenTime}.\n */\n #lastTimeoutCheckAt: number;\n\n /**\n * Resolver used to signal waiting callers when the state changes.\n */\n #stateChangeResolver: Resolver<ConnectionManagerState> = resolver();\n\n constructor(options: ConnectionManagerOptions) {\n super();\n\n this.#disconnectTimeout = options.disconnectTimeout;\n this.#timeoutCheckIntervalMs =\n options.timeoutCheckIntervalMs ?? DEFAULT_TIMEOUT_CHECK_INTERVAL_MS;\n // The connecting window, and the timer that enforces it, start in\n // initialized(), once the local store is loaded.\n this.#state = {name: ConnectionStatus.Initializing};\n this.#lastTimeoutCheckAt = Date.now();\n }\n\n get state(): ConnectionManagerState {\n return this.#state;\n }\n\n /**\n * Returns true if the current state is equal to the given status.\n */\n is(status: ConnectionStatus): boolean {\n return this.#state.name === status;\n }\n\n /**\n * Returns true if the current state is a terminal state\n * that can be recovered from by calling connect().\n */\n isInTerminalState(): boolean {\n return ConnectionManager.isTerminalState(this.#state);\n }\n\n /**\n * Returns true if the given status is a terminal state\n * that can be recovered from by calling connect().\n */\n static isTerminalState(\n state: ConnectionManagerState,\n ): state is TerminalConnectionManagerState {\n return (TERMINAL_STATES as readonly ConnectionStatus[]).includes(\n state.name,\n );\n }\n\n /**\n * Returns true if the run loop should continue.\n * The run loop continues in all states except closed.\n * In needs-auth and error states, the run loop pauses and waits for connect() to be called.\n */\n shouldContinueRunLoop(): boolean {\n return this.#state.name !== ConnectionStatus.Closed;\n }\n\n /**\n * Waits for the next state change.\n * @returns A promise that resolves when the next state change occurs.\n */\n waitForStateChange(): Promise<ConnectionManagerState> {\n return this.#nextStatePromise();\n }\n\n requestConnect(): void {\n if (this.#connectRequestResolver === undefined) {\n return;\n }\n this.#connectRequestResolver.resolve();\n this.#connectRequestResolver = undefined;\n }\n\n waitForConnectRequest(): Promise<void> {\n return this.#connectRequestResolver === undefined\n ? Promise.resolve()\n : this.#connectRequestResolver.promise;\n }\n\n #consumeConnectRequest(): boolean {\n if (this.#connectRequestResolver !== undefined) {\n return false;\n }\n this.#connectRequestResolver = resolver();\n return true;\n }\n\n /**\n * Consume a pending connect request and resume connecting.\n *\n * @returns true if a pending request was handled.\n */\n resumeFromConnectRequest(): boolean {\n if (!this.isInTerminalState()) {\n return false;\n }\n if (!this.#consumeConnectRequest()) {\n return false;\n }\n this.connecting();\n return true;\n }\n\n /**\n * Transition from initializing to connecting, once the local store is\n * loaded.\n *\n * This is where the connecting window starts. Loading the local store can\n * take tens of seconds on a slow device with a large replica, and none of\n * that says anything about whether the server is reachable, so it is not\n * charged against the disconnect timeout. No connect attempt has been made\n * yet, so `attempt` is 0.\n *\n * A no-op in any other state: something else, like `close()`, has already\n * moved the state on.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n initialized(): {nextStatePromise: Promise<ConnectionManagerState>} {\n if (this.#state.name !== ConnectionStatus.Initializing) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n const now = Date.now();\n this.#connectingStartedAt = now;\n this.#state = {\n name: ConnectionStatus.Connecting,\n attempt: 0,\n disconnectAt: now + this.#disconnectTimeout,\n };\n // Start the interval before publishing: a subscriber may close the manager\n // synchronously, and closed() can only stop an interval that exists.\n this.#maybeStartTimeoutInterval();\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to connecting state.\n *\n * This starts the timeout timer, but if we've entered disconnected state,\n * we stay there and continue retrying.\n *\n * If we're disconnected due to a hidden tab, allow a transition back to\n * connecting when visibility returns.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n connecting(reason?: ZeroError): {\n nextStatePromise: Promise<ConnectionManagerState>;\n } {\n // cannot transition from closed to any other status\n if (this.#state.name === ConnectionStatus.Closed) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n const isHiddenDisconnect =\n this.#state.name === ConnectionStatus.Disconnected &&\n this.#state.reason.kind === ClientErrorKind.Hidden;\n\n // we cannot intentionally transition from disconnected to connecting\n // unless the disconnection was due to a hidden tab\n if (\n this.#state.name === ConnectionStatus.Disconnected &&\n !isHiddenDisconnect\n ) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n const now = Date.now();\n\n if (isHiddenDisconnect) {\n // Reset the retry window after a hidden-tab disconnect.\n this.#connectingStartedAt = undefined;\n }\n\n // If we're already connecting, increment the attempt counter\n if (this.#state.name === ConnectionStatus.Connecting) {\n this.#state = {\n ...this.#state,\n attempt: this.#state.attempt + 1,\n reason,\n };\n // See initialized() for why the interval starts before publishing.\n this.#maybeStartTimeoutInterval();\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n // Starting a new connecting session\n // If #connectingStartedAt is undefined, this is a fresh start - set it to now\n // If it's already set, we're retrying within the same retry window, so keep it\n if (this.#connectingStartedAt === undefined) {\n this.#connectingStartedAt = now;\n }\n\n const disconnectAt = this.#connectingStartedAt + this.#disconnectTimeout;\n\n this.#state = {\n name: ConnectionStatus.Connecting,\n attempt: 1,\n disconnectAt,\n reason,\n };\n // Start the interval before publishing: a subscriber may close the manager\n // synchronously, and closed() can only stop an interval that exists.\n this.#maybeStartTimeoutInterval();\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to connected state.\n * This resets the connecting timeout timer.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n connected(): {nextStatePromise: Promise<ConnectionManagerState>} {\n // cannot transition from closed to any other status\n if (this.#state.name === ConnectionStatus.Closed) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Already connected, no-op\n if (this.#state.name === ConnectionStatus.Connected) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Reset the timeout timer on successful connection\n this.#connectingStartedAt = undefined;\n this.#maybeStopTimeoutInterval();\n\n this.#state = {\n name: ConnectionStatus.Connected,\n };\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to disconnected state.\n * This is called when the timeout expires.\n * The run loop will continue trying to reconnect.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n disconnected(reason: DisconnectedReason): {\n nextStatePromise: Promise<ConnectionManagerState>;\n } {\n // cannot transition from closed to any other status\n if (this.#state.name === ConnectionStatus.Closed) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Already disconnected, no-op\n if (this.#state.name === ConnectionStatus.Disconnected) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // When transitioning from connected to disconnected, we've lost a connection\n // we previously had. Clear the timeout timer so we can start a fresh timeout window.\n if (this.#state.name === ConnectionStatus.Connected) {\n this.#connectingStartedAt = undefined;\n }\n // When transitioning from connecting to disconnected (e.g., due to timeout),\n // we keep the start time to maintain the context that we've been trying for a while.\n\n this.#maybeStopTimeoutInterval();\n\n this.#state = {\n name: ConnectionStatus.Disconnected,\n reason,\n };\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to needs-auth state.\n * This pauses the run loop until connect() is called with new credentials.\n * Resets the retry window and attempt counter.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n needsAuth(reason: AuthError): {\n nextStatePromise: Promise<ConnectionManagerState>;\n } {\n // cannot transition from closed to any other status\n if (this.#state.name === ConnectionStatus.Closed) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Already in needs-auth state, no-op\n if (this.#state.name === ConnectionStatus.NeedsAuth) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Reset the timeout timer and connecting start time\n this.#connectingStartedAt = undefined;\n this.#maybeStopTimeoutInterval();\n\n this.#state = {\n name: ConnectionStatus.NeedsAuth,\n reason,\n };\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to error state.\n * This pauses the run loop until connect() is called.\n * Resets the retry window and attempt counter.\n *\n * @returns An object containing a promise that resolves on the next state change.\n */\n error(reason: ZeroError): {\n nextStatePromise: Promise<ConnectionManagerState>;\n } {\n // cannot transition from closed to any other status\n if (this.#state.name === ConnectionStatus.Closed) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Already in error state, no-op\n if (this.#state.name === ConnectionStatus.Error) {\n return {nextStatePromise: this.#nextStatePromise()};\n }\n\n // Reset the timeout timer and connecting start time\n this.#connectingStartedAt = undefined;\n this.#maybeStopTimeoutInterval();\n\n this.#state = {\n name: ConnectionStatus.Error,\n reason,\n };\n const nextStatePromise = this.#publishStateAndGetPromise();\n return {nextStatePromise};\n }\n\n /**\n * Transition to closed state.\n * This is terminal - no further transitions are allowed.\n */\n closed() {\n // Already closed, no-op\n if (this.#state.name === ConnectionStatus.Closed) {\n return;\n }\n\n this.#connectingStartedAt = undefined;\n this.#maybeStopTimeoutInterval();\n\n this.#state = {\n name: ConnectionStatus.Closed,\n reason: new ClientError({\n kind: ClientErrorKind.ClientClosed,\n message: 'Zero was explicitly closed by calling zero.close()',\n }),\n };\n this.#publishState();\n this.cleanup();\n return;\n }\n\n override cleanup = (): void => {\n this._listeners.clear();\n this.#resolveNextStateWaiters();\n };\n\n #resolveNextStateWaiters(): void {\n this.#stateChangeResolver.resolve(this.#state);\n this.#stateChangeResolver = resolver();\n }\n\n #publishState(): void {\n this.notify(this.#state);\n this.#resolveNextStateWaiters();\n }\n\n #nextStatePromise(): Promise<ConnectionManagerState> {\n return this.#stateChangeResolver.promise;\n }\n\n #publishStateAndGetPromise(): Promise<ConnectionManagerState> {\n this.#publishState();\n return this.#nextStatePromise();\n }\n\n /**\n * Pushes the disconnect deadline forward by however long the event loop was\n * not running.\n *\n * `disconnectAt` is an absolute wall-clock instant, but what it is meant to\n * bound is time spent *actually retrying*. Whenever the loop is frozen --- a\n * suspended React Native app, a sleeping laptop, a throttled background tab\n * --- wall-clock keeps advancing while no reconnect attempt can run, so\n * without this the first tick after resume reports `Offline` having given the\n * client zero live seconds to connect.\n *\n * This interval is its own detector: a tick that lands late by more than a\n * full period means we were not running in between. A merely busy JS thread\n * can also delay a tick, and we cannot tell the two apart from lateness\n * alone, but the costs are asymmetric --- crediting a busy period back just\n * grants a bit more time to connect, whereas failing to credit a real freeze\n * produces a user-visible bogus disconnect --- so we credit.\n *\n * The whole gap is credited, not `elapsed` minus a period. An overdue\n * `setInterval` callback runs as soon as the loop resumes rather than waiting\n * out another period, so `elapsed` is already about the frozen duration;\n * netting off a period would charge up to a full period of frozen time\n * against the deadline. Freezing just after a tick would then leave the\n * deadline exactly at `now` on resume and disconnect immediately, which is\n * the very thing this is here to prevent.\n */\n #creditFrozenTime(): void {\n const now = Date.now();\n const elapsed = now - this.#lastTimeoutCheckAt;\n this.#lastTimeoutCheckAt = now;\n\n // Ordinary scheduling jitter, not a freeze.\n if (elapsed <= 2 * this.#timeoutCheckIntervalMs) {\n return;\n }\n const frozenMs = elapsed;\n\n // Advance the window's origin so that a later `connecting()` that starts a\n // fresh session recomputes a deadline that also excludes the frozen time.\n if (this.#connectingStartedAt !== undefined) {\n this.#connectingStartedAt += frozenMs;\n }\n\n if (this.#state.name === ConnectionStatus.Connecting) {\n // Deliberately not published: the status has not changed, and waking the\n // run loop's `waitForStateChange` racers here would interrupt an\n // in-flight connect attempt for no reason. Readers of `state` still see\n // the corrected deadline.\n this.#state = {\n ...this.#state,\n disconnectAt: this.#state.disconnectAt + frozenMs,\n };\n }\n }\n\n /**\n * Check if we should transition from connecting to disconnected due to timeout.\n * Returns true if the transition happened.\n */\n #checkTimeout(): boolean {\n if (this.#state.name !== ConnectionStatus.Connecting) {\n return false;\n }\n\n const now = Date.now();\n if (now >= this.#state.disconnectAt) {\n this.disconnected(\n new ClientError({\n kind: ClientErrorKind.Offline,\n message: `Zero was unable to connect for ${Math.floor(this.#disconnectTimeout / 1_000)} seconds and was disconnected`,\n }),\n );\n return true;\n }\n\n return false;\n }\n\n #maybeStartTimeoutInterval(): void {\n if (this.#timeoutInterval !== undefined) {\n return;\n }\n this.#lastTimeoutCheckAt = Date.now();\n this.#timeoutInterval = setInterval(() => {\n this.#creditFrozenTime();\n this.#checkTimeout();\n }, this.#timeoutCheckIntervalMs);\n }\n\n #maybeStopTimeoutInterval(): void {\n if (this.#timeoutInterval === undefined) {\n return;\n }\n clearInterval(this.#timeoutInterval);\n this.#timeoutInterval = undefined;\n }\n}\n\n/**\n * Used to trigger the catch block when a terminal state is reached.\n *\n * @param state - The current connection state.\n */\nexport const throwIfConnectionError = (state: ConnectionManagerState) => {\n if (\n ConnectionManager.isTerminalState(state) ||\n state.name === ConnectionStatus.Closed ||\n ((state.name === ConnectionStatus.Connecting ||\n state.name === ConnectionStatus.Disconnected) &&\n state.reason)\n ) {\n if (\n isClientError(state.reason) &&\n (state.reason.kind === ClientErrorKind.ConnectTimeout ||\n state.reason.kind === ClientErrorKind.AbruptClose ||\n state.reason.kind === ClientErrorKind.CleanClose ||\n state.reason.kind === ClientErrorKind.Hidden)\n ) {\n return;\n }\n throw state.reason;\n }\n};\n"],"mappings":";;;;;;AAaA,IAAM,oCAAoC;AA6C1C,IAAM,kBAAkB,CACtB,WACA,KACF;AAQA,IAAa,oBAAb,MAAa,0BAA0B,aAAqC;CAC1E;CACA,0BAAsD,SAAS;;;;;;CAO/D;;;;;CAMA;;;;;CAMA;;;;CAKA;;;;;CAMA;;;;CAKA,uBAAyD,SAAS;CAElE,YAAY,SAAmC;EAC7C,MAAM;EAEN,KAAKA,qBAAqB,QAAQ;EAClC,KAAKC,0BACH,QAAQ,0BAA0B;EAGpC,KAAKC,SAAS,EAAC,MAAM,aAA6B;EAClD,KAAKC,sBAAsB,KAAK,IAAI;CACtC;CAEA,IAAI,QAAgC;EAClC,OAAO,KAAKD;CACd;;;;CAKA,GAAG,QAAmC;EACpC,OAAO,KAAKA,OAAO,SAAS;CAC9B;;;;;CAMA,oBAA6B;EAC3B,OAAO,kBAAkB,gBAAgB,KAAKA,MAAM;CACtD;;;;;CAMA,OAAO,gBACL,OACyC;EACzC,OAAQ,gBAAgD,SACtD,MAAM,IACR;CACF;;;;;;CAOA,wBAAiC;EAC/B,OAAO,KAAKA,OAAO,SAAS;CAC9B;;;;;CAMA,qBAAsD;EACpD,OAAO,KAAKE,kBAAkB;CAChC;CAEA,iBAAuB;EACrB,IAAI,KAAKC,4BAA4B,KAAA,GACnC;EAEF,KAAKA,wBAAwB,QAAQ;EACrC,KAAKA,0BAA0B,KAAA;CACjC;CAEA,wBAAuC;EACrC,OAAO,KAAKA,4BAA4B,KAAA,IACpC,QAAQ,QAAQ,IAChB,KAAKA,wBAAwB;CACnC;CAEA,yBAAkC;EAChC,IAAI,KAAKA,4BAA4B,KAAA,GACnC,OAAO;EAET,KAAKA,0BAA0B,SAAS;EACxC,OAAO;CACT;;;;;;CAOA,2BAAoC;EAClC,IAAI,CAAC,KAAK,kBAAkB,GAC1B,OAAO;EAET,IAAI,CAAC,KAAKC,uBAAuB,GAC/B,OAAO;EAET,KAAK,WAAW;EAChB,OAAO;CACT;;;;;;;;;;;;;;;;CAiBA,cAAmE;EACjE,IAAI,KAAKJ,OAAO,SAAS,gBACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAGpD,MAAM,MAAM,KAAK,IAAI;EACrB,KAAKG,uBAAuB;EAC5B,KAAKL,SAAS;GACZ,MAAM;GACN,SAAS;GACT,cAAc,MAAM,KAAKF;EAC3B;EAGA,KAAKQ,2BAA2B;EAEhC,OAAO,EAAC,kBADiB,KAAKC,2BACtB,EAAgB;CAC1B;;;;;;;;;;;;CAaA,WAAW,QAET;EAEA,IAAI,KAAKP,OAAO,SAAS,UACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAGpD,MAAM,qBACJ,KAAKF,OAAO,SAAS,kBACrB,KAAKA,OAAO,OAAO,SAAS;EAI9B,IACE,KAAKA,OAAO,SAAS,kBACrB,CAAC,oBAED,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAGpD,MAAM,MAAM,KAAK,IAAI;EAErB,IAAI,oBAEF,KAAKG,uBAAuB,KAAA;EAI9B,IAAI,KAAKL,OAAO,SAAS,cAA6B;GACpD,KAAKA,SAAS;IACZ,GAAG,KAAKA;IACR,SAAS,KAAKA,OAAO,UAAU;IAC/B;GACF;GAEA,KAAKM,2BAA2B;GAEhC,OAAO,EAAC,kBADiB,KAAKC,2BACtB,EAAgB;EAC1B;EAKA,IAAI,KAAKF,yBAAyB,KAAA,GAChC,KAAKA,uBAAuB;EAG9B,MAAM,eAAe,KAAKA,uBAAuB,KAAKP;EAEtD,KAAKE,SAAS;GACZ,MAAM;GACN,SAAS;GACT;GACA;EACF;EAGA,KAAKM,2BAA2B;EAEhC,OAAO,EAAC,kBADiB,KAAKC,2BACtB,EAAgB;CAC1B;;;;;;;CAQA,YAAiE;EAE/D,IAAI,KAAKP,OAAO,SAAS,UACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,IAAI,KAAKF,OAAO,SAAS,aACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,KAAKG,uBAAuB,KAAA;EAC5B,KAAKG,0BAA0B;EAE/B,KAAKR,SAAS,EACZ,MAAM,UACR;EAEA,OAAO,EAAC,kBADiB,KAAKO,2BACtB,EAAgB;CAC1B;;;;;;;;CASA,aAAa,QAEX;EAEA,IAAI,KAAKP,OAAO,SAAS,UACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,IAAI,KAAKF,OAAO,SAAS,gBACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAKpD,IAAI,KAAKF,OAAO,SAAS,aACvB,KAAKK,uBAAuB,KAAA;EAK9B,KAAKG,0BAA0B;EAE/B,KAAKR,SAAS;GACZ,MAAM;GACN;EACF;EAEA,OAAO,EAAC,kBADiB,KAAKO,2BACtB,EAAgB;CAC1B;;;;;;;;CASA,UAAU,QAER;EAEA,IAAI,KAAKP,OAAO,SAAS,UACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,IAAI,KAAKF,OAAO,SAAS,cACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,KAAKG,uBAAuB,KAAA;EAC5B,KAAKG,0BAA0B;EAE/B,KAAKR,SAAS;GACZ,MAAM;GACN;EACF;EAEA,OAAO,EAAC,kBADiB,KAAKO,2BACtB,EAAgB;CAC1B;;;;;;;;CASA,MAAM,QAEJ;EAEA,IAAI,KAAKP,OAAO,SAAS,UACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,IAAI,KAAKF,OAAO,SAAS,SACvB,OAAO,EAAC,kBAAkB,KAAKE,kBAAkB,EAAC;EAIpD,KAAKG,uBAAuB,KAAA;EAC5B,KAAKG,0BAA0B;EAE/B,KAAKR,SAAS;GACZ,MAAM;GACN;EACF;EAEA,OAAO,EAAC,kBADiB,KAAKO,2BACtB,EAAgB;CAC1B;;;;;CAMA,SAAS;EAEP,IAAI,KAAKP,OAAO,SAAS,UACvB;EAGF,KAAKK,uBAAuB,KAAA;EAC5B,KAAKG,0BAA0B;EAE/B,KAAKR,SAAS;GACZ,MAAM;GACN,QAAQ,IAAI,YAAY;IACtB,MAAM;IACN,SAAS;GACX,CAAC;EACH;EACA,KAAKS,cAAc;EACnB,KAAK,QAAQ;CAEf;CAEA,gBAA+B;EAC7B,KAAK,WAAW,MAAM;EACtB,KAAKC,yBAAyB;CAChC;CAEA,2BAAiC;EAC/B,KAAKC,qBAAqB,QAAQ,KAAKX,MAAM;EAC7C,KAAKW,uBAAuB,SAAS;CACvC;CAEA,gBAAsB;EACpB,KAAK,OAAO,KAAKX,MAAM;EACvB,KAAKU,yBAAyB;CAChC;CAEA,oBAAqD;EACnD,OAAO,KAAKC,qBAAqB;CACnC;CAEA,6BAA8D;EAC5D,KAAKF,cAAc;EACnB,OAAO,KAAKP,kBAAkB;CAChC;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4BA,oBAA0B;EACxB,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,UAAU,MAAM,KAAKD;EAC3B,KAAKA,sBAAsB;EAG3B,IAAI,WAAW,IAAI,KAAKF,yBACtB;EAEF,MAAM,WAAW;EAIjB,IAAI,KAAKM,yBAAyB,KAAA,GAChC,KAAKA,wBAAwB;EAG/B,IAAI,KAAKL,OAAO,SAAS,cAKvB,KAAKA,SAAS;GACZ,GAAG,KAAKA;GACR,cAAc,KAAKA,OAAO,eAAe;EAC3C;CAEJ;;;;;CAMA,gBAAyB;EACvB,IAAI,KAAKA,OAAO,SAAS,cACvB,OAAO;EAIT,IADY,KAAK,IACb,KAAO,KAAKA,OAAO,cAAc;GACnC,KAAK,aACH,IAAI,YAAY;IACd,MAAM;IACN,SAAS,kCAAkC,KAAK,MAAM,KAAKF,qBAAqB,GAAK,EAAE;GACzF,CAAC,CACH;GACA,OAAO;EACT;EAEA,OAAO;CACT;CAEA,6BAAmC;EACjC,IAAI,KAAKc,qBAAqB,KAAA,GAC5B;EAEF,KAAKX,sBAAsB,KAAK,IAAI;EACpC,KAAKW,mBAAmB,kBAAkB;GACxC,KAAKC,kBAAkB;GACvB,KAAKC,cAAc;EACrB,GAAG,KAAKf,uBAAuB;CACjC;CAEA,4BAAkC;EAChC,IAAI,KAAKa,qBAAqB,KAAA,GAC5B;EAEF,cAAc,KAAKA,gBAAgB;EACnC,KAAKA,mBAAmB,KAAA;CAC1B;AACF;;;;;;AAOA,IAAa,0BAA0B,UAAkC;CACvE,IACE,kBAAkB,gBAAgB,KAAK,KACvC,MAAM,SAAS,aACb,MAAM,SAAS,gBACf,MAAM,SAAS,mBACf,MAAM,QACR;EACA,IACE,cAAc,MAAM,MAAM,MACzB,MAAM,OAAO,SAAS,oBACrB,MAAM,OAAO,SAAS,iBACtB,MAAM,OAAO,SAAS,gBACtB,MAAM,OAAO,SAAS,WAExB;EAEF,MAAM,MAAM;CACd;AACF"}
|