@rocicorp/zero 1.11.0-canary.31 → 1.11.0-canary.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/analyze-query/src/analyze-cli.js +2 -2
- package/out/analyze-query/src/analyze-cli.js.map +1 -1
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/server/slot-keeper.d.ts +26 -0
- package/out/zero-cache/src/server/slot-keeper.d.ts.map +1 -0
- package/out/zero-cache/src/server/slot-keeper.js +175 -0
- package/out/zero-cache/src/server/slot-keeper.js.map +1 -0
- package/out/zero-cache/src/server/worker-urls.d.ts +1 -0
- package/out/zero-cache/src/server/worker-urls.d.ts.map +1 -1
- package/out/zero-cache/src/server/worker-urls.js +2 -1
- package/out/zero-cache/src/server/worker-urls.js.map +1 -1
- package/out/zero-cache/src/services/change-source/custom/change-source.js +2 -2
- package/out/zero-cache/src/services/change-source/custom/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js +4 -3
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +2 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts +35 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js +106 -22
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js +31 -9
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +2 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/connection-context-manager.d.ts +3 -2
- package/out/zero-cache/src/services/view-syncer/connection-context-manager.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/connection-context-manager.js +1 -1
- package/out/zero-cache/src/services/view-syncer/connection-context-manager.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js.map +1 -1
- package/out/zero-cache/src/types/pg.js +1 -1
- package/out/zero-cache/src/types/processes.d.ts +3 -0
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/workers/connect-params.d.ts +6 -15
- package/out/zero-cache/src/workers/connect-params.d.ts.map +1 -1
- package/out/zero-cache/src/workers/connect-params.js +14 -15
- package/out/zero-cache/src/workers/connect-params.js.map +1 -1
- package/out/zero-cache/src/workers/connection.d.ts.map +1 -1
- package/out/zero-cache/src/workers/connection.js +4 -4
- package/out/zero-cache/src/workers/connection.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +2 -2
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero.js +2 -2
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-protocol/src/analyze-query-result.d.ts +1 -1
- package/out/zero-protocol/src/analyze-query-result.js.map +1 -1
- package/out/zero-protocol/src/connect.d.ts +9 -12
- package/out/zero-protocol/src/connect.d.ts.map +1 -1
- package/out/zero-protocol/src/connect.js +19 -16
- package/out/zero-protocol/src/connect.js.map +1 -1
- package/out/zero-protocol/src/poke.d.ts +3 -3
- package/out/zero-protocol/src/poke.js +1 -1
- package/out/zero-protocol/src/poke.js.map +1 -1
- package/out/zero-protocol/src/{feature-flag-enum.d.ts → protocol-flag-enum.d.ts} +1 -1
- package/out/zero-protocol/src/protocol-flag-enum.d.ts.map +1 -0
- package/out/zero-protocol/src/protocol-flag-enum.js +10 -0
- package/out/zero-protocol/src/protocol-flag-enum.js.map +1 -0
- package/out/zero-protocol/src/protocol-flag.d.ts +14 -0
- package/out/zero-protocol/src/protocol-flag.d.ts.map +1 -0
- package/out/zero-protocol/src/protocol-flag.js +8 -0
- package/out/zero-protocol/src/protocol-flag.js.map +1 -0
- package/out/zero-protocol/src/protocol-version.d.ts +3 -3
- package/out/zero-protocol/src/protocol-version.js.map +1 -1
- package/package.json +1 -1
- package/out/shared/src/packed-boolean-map.d.ts +0 -7
- package/out/shared/src/packed-boolean-map.d.ts.map +0 -1
- package/out/shared/src/packed-boolean-map.js +0 -36
- package/out/shared/src/packed-boolean-map.js.map +0 -1
- package/out/zero-protocol/src/feature-flag-enum.d.ts.map +0 -1
- package/out/zero-protocol/src/feature-flag-enum.js +0 -10
- package/out/zero-protocol/src/feature-flag-enum.js.map +0 -1
- package/out/zero-protocol/src/feature-flag.d.ts +0 -5
- package/out/zero-protocol/src/feature-flag.d.ts.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n PG_CONFIGURATION_LIMIT_EXCEEDED,\n PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n createReplicationSessionFor,\n keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n InitialSync,\n Restore,\n type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n createReplica,\n getReplicaState,\n getRestoreCandidates,\n metadataPublicationName,\n replicationSlotExpression,\n replicationSlotPrefix,\n type BackupOptions,\n type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n slot_name: string;\n consistent_point: string;\n snapshot_name: string;\n output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n slot: ReplicationSlot;\n capturedSnapshot: T;\n initialSession: Readable;\n replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n slot: string;\n reservation: Readable;\n replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n slotName: string;\n\n // Note: ignored if pgVersion < PG_17.\n failover?: boolean;\n\n // Create a temporary slot (i.e. not persisted, and automatically\n // cleaned up when the replication session ends).\n temporary?: boolean;\n\n // For overriding in tests.\n lockTimeout?: number;\n\n // A (non-replication) connection to the same upstream instance, used to\n // log what slot creation was waiting on if it times out.\n diagnosticsDB?: PostgresDB | undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n// so all commands must be sent using sql.unsafe(). This is technically safe\n// because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n lc: LogContext,\n session: postgres.Sql,\n {\n slotName,\n failover,\n temporary,\n lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n diagnosticsDB,\n }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n // does NOT apply to replication commands, but lock_timeout does (it governs\n // the heavyweight lock wait inside LockAcquire). Setting it here causes\n // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n // walsender, rather than relying solely on the client-side orTimeout\n // which can leave an orphaned backend.\n //\n // An orphaned walsender is actively harmful: by this point the replication\n // slot has already been created and is pinning WAL retention and catalog_xmin.\n // Worse, the slot is marked `active` (the walsender PID is still alive), so\n // the existing cleanup code (which drops inactive slots on retry) can't\n // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n // fires (~2h default) or the blocking transaction finishes.\n await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n const {pgVersion, pid: walsenderPID} = (\n await session.unsafe<{pgVersion: number; pid: number}[]>(`\n SELECT current_setting('server_version_num') as \"pgVersion\",\n pg_backend_pid() as \"pid\";\n `)\n )[0];\n\n const maybeTemporary = temporary ? 'TEMPORARY' : '';\n const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n if (raced === 'timed-out') {\n if (diagnosticsDB) {\n await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n }\n // Create slot can block indefinitely waiting for old transactions. End\n // this connection in the background and fail fast so the process restarts.\n void session\n .end()\n .catch(e =>\n lc.warn?.(`Error closing timed out replication slot session`, e),\n );\n throw new Error(\n `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n );\n }\n const [slot] = raced;\n lc.info?.(`Created replication slot ${slotName}`, slot);\n return slot;\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/**\n * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n *\n * Query text is deliberately omitted since it can contain row values.\n *\n * Exported for testing.\n */\nexport async function logSlotCreationBlockers(\n lc: LogContext,\n sql: PostgresDB,\n walsenderPID: number,\n slotName: string,\n) {\n try {\n const result = await orTimeout(\n Promise.all([\n sql<JSONObject[]> /*sql*/ `\n SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n pg_blocking_pids(pid) as \"blockingPIDs\"\n FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n sql<JSONObject[]> /*sql*/ `\n SELECT pid, datname, usename, application_name as \"applicationName\",\n backend_type as \"backendType\", state,\n wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n backend_xid::text as xid,\n xact_start::text as \"xactStart\",\n extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n FROM pg_stat_activity\n WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n ORDER BY xact_start NULLS LAST\n LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n ]),\n SLOT_DIAGNOSTICS_TIMEOUT_MS,\n );\n if (result === 'timed-out') {\n lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n return;\n }\n const [[walsender], transactions] = result;\n lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n walsender: walsender ?? null,\n transactions,\n });\n } catch (e) {\n lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n }\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n * replica slot management. This is the same lock that cleanup logic\n * acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n * replication session. The API of CREATE_REPLICATION_SLOT is such\n * that it cannot be done in a transaction, and cannot be followed by\n * any writes, or else its snapshot (which is needed for initial sync)\n * would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n * in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n * will not use the same name for the replication slot (which is selected\n * from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n * rank) will not delete the new slot during their cleanup logic, since\n * the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n lc: LogContext,\n sql: PostgresDB,\n sessionName: string,\n shard: ShardID,\n epoch: number,\n replicaID: string,\n failover: boolean,\n backupOptions: BackupOptions,\n captureSnapshot: (snapshot: string) => Promise<T>,\n stage: ReplicaStage,\n): Promise<ReplicationSlotResult<T>> {\n // Note: The replicationSession is closed by keepSlotActiveUntilTakenOver,\n // and only closed in this function if an error occurrs.\n const replicationSession = createReplicationSessionFor(sql, sessionName);\n const lockName = replicationSlotManagementLock(shard);\n const slotPoolPrefix = replicationSlotPrefix(shard);\n\n for (let first = true; ; first = false) {\n await dropUnclaimedSlots(lc, sql, shard);\n\n let slotName: string | undefined;\n try {\n return await runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n if (stage === InitialSync) {\n // With the lock acquired, ensure that only one initial sync is\n // active at a time. getRestoreCandidates() orders its results\n // with stage = InitialSync (and active = true) first.\n const others = await getRestoreCandidates(tx, shard, epoch);\n lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n if (others.length) {\n const [{id, stage, active}] = others;\n if (stage === InitialSync && active) {\n throw new AutoResetSignal(\n `another replica (${id}) is performing initial sync`,\n );\n }\n }\n }\n\n // Pick an available slotName from the slotPoolPrefix pool.\n const names = await tx<{name: string}[]> /*sql*/ `\n SELECT slot_name as name FROM pg_replication_slots\n WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n `.values();\n const inUse = new Set(names.flat());\n for (let next = 0; ; next++) {\n const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n if (!inUse.has(candidateName)) {\n slotName = candidateName;\n break;\n }\n }\n\n const slot = await createReplicationSlot(lc, replicationSession, {\n slotName,\n failover,\n diagnosticsDB: sql,\n });\n const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n const initialSession = await keepSlotActiveUntilTakenOver(\n lc,\n replicationSession,\n slot.slot_name,\n metadataPublicationName(shard.appID, shard.shardNum),\n toBigInt(slot.consistent_point),\n );\n\n await createReplica(\n tx,\n shard,\n replicaID,\n slot.slot_name,\n epoch,\n stage === InitialSync\n ? toStateVersionString(slot.consistent_point)\n : '', // initialized with initRestoreReplica\n backupOptions,\n stage,\n );\n\n const replica = must(\n await getReplicaState(tx, shard, replicaID),\n `replica ${replicaID} was not created`,\n );\n\n return {slot, capturedSnapshot, initialSession, replica};\n });\n } catch (e) {\n if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n // Some Postgres variants (e.g. Google Cloud SQL) require that\n // the user have the REPLICATION role in order to create a slot.\n // Note that this must be done by the upstreamDB connection, and\n // does not work in the replicationSession itself.\n await sql`ALTER ROLE current_user WITH REPLICATION`;\n lc.info?.(`Added the REPLICATION role to database user`);\n continue;\n }\n // Note: This is currently manually tested since max_replication_slots\n // is a PG startup parameter that other tests depend on.\n // TODO: Figure out a way to unit test this (with the full PG setup).\n if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n lc.warn?.(\n `Reached max replication slots. Attempting to clean up unused slots`,\n e,\n );\n // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n await sql`\n DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n continue; // then let dropUnclaimedSlots() perform its cleanup\n }\n // Otherwise, clean up any created slot if something went wrong. The\n // slot may not exist (e.g. if its creation failed), and its name may\n // have been reused since the lock was released, so leave it to\n // dropUnclaimedSlots(), which only drops slots without a replica.\n if (slotName) {\n lc.warn?.(`deleting slot ${slotName} due to error`, e);\n await replicationSession.end();\n await dropUnclaimedSlots(lc, sql, shard);\n }\n throw e;\n }\n }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n sessionName: string,\n {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n const replicationSession = createReplicationSessionFor(sql, sessionName);\n const lockName = replicationSlotManagementLock(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n try {\n const reserved = await runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n SELECT active, confirmed_flush_lsn as lsn\n FROM pg_replication_slots\n JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n `;\n if (replicas.length === 0) {\n lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n return null;\n }\n const [{active, lsn}] = replicas;\n if (active) {\n lc.warn?.(\n `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n );\n return null;\n }\n // Mark the replica as Restoring.\n await tx`\n UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n `;\n const replica = must(\n await getReplicaState(tx, shard, replicaID),\n `replica ${replicaID} disappeared`,\n );\n\n return {\n slot,\n replica,\n reservation: await keepSlotActiveUntilTakenOver(\n lc,\n replicationSession,\n slot,\n metadataPublicationName(shard.appID, shard.shardNum),\n toBigInt(lsn),\n ),\n };\n });\n lc.info?.(`successfully claimed slot ${slot}`);\n return reserved;\n } catch (e) {\n lc.error?.(`unable to claim replication slot ${slot}`, e);\n await replicationSession.end();\n return null;\n }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n slots: string[],\n) {\n const lockName = replicationSlotManagementLock(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n return runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const dropped = await tx<{slot: string; replicaID: string | null}[]>\n /*sql*/ `\n SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name IN ${tx(slots)} AND NOT active;\n `;\n if (dropped.length) {\n lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n dropped,\n });\n }\n\n const replicas = dropped\n .map(({replicaID}) => replicaID)\n .filter(id => id !== null);\n if (replicas.length) {\n // Delete replicas associated with the inactive (and now dropped) slots.\n await tx\n /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n }\n });\n}\n\nfunction dropUnclaimedSlots(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n // The slot / replica cleanup happens within a transaction while holding\n // the replication slot management lock for this shard, to ensure that no\n // slot that belongs to a newer replica is dropped.\n const lockName = replicationSlotManagementLock(shard);\n const slotExpression = replicationSlotExpression(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n return runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const dropped = await tx /*sql*/ `\n SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name LIKE ${slotExpression} \n AND NOT active\n AND replica.id IS NULL;\n `;\n if (dropped.length) {\n lc.info?.(`dropped inactive replication slots`, {dropped});\n }\n\n // Conversely, delete replicas whose slots no longer exist, e.g. because\n // they were dropped outside of zero-cache's control (such as in an\n // upstream failover). Slot names are reused, so such rows would otherwise\n // be associated with a new slot of the same name. This is done (and\n // committed) before the slot is created, both so that the rows are never\n // associated with it, and because CREATE_REPLICATION_SLOT waits for\n // transactions that have written (i.e. been assigned an xid).\n const orphaned = await tx<{id: string}[]> /*sql*/ `\n DELETE FROM ${tx(replicasTable)} replica\n WHERE NOT EXISTS (\n SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n )\n RETURNING id;\n `;\n if (orphaned.length) {\n lc.warn?.(`deleted replicas whose slots no longer exist`, {\n replicas: orphaned.map(({id}) => id),\n });\n }\n\n const remaining = await tx<\n {slot: string; pid: number | null; id: string | null}[]\n > /*sql*/ `\n SELECT slot_name as slot, active_pid as pid, replica.id as id\n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name LIKE ${slotExpression};\n `;\n if (remaining.length) {\n lc.info?.(`remaining replication slots`, {remaining});\n }\n\n let active = 0;\n let draining = 0;\n for (const {id} of remaining) {\n if (id === null) {\n draining++;\n } else {\n active++;\n }\n }\n\n return {\n dropped: dropped.length,\n active,\n draining,\n };\n });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n n++; // Adjust for 0-based indexing\n\n let suffix = '';\n while (n > 0) {\n n--;\n suffix = ALPHABET[n % 26] + suffix;\n n = Math.floor(n / 26);\n }\n return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;AAmFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,GACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAIhE,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EACzB,IAAI,eACF,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAIzE,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACmC;CAGnC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAElD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAEvC,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAE1D,IAAI,UAAA,GAAuB;KAIzB,MAAM,SAAS,MAAM,qBAAqB,IAAI,OAAO,KAAK;KAC1D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;KAClE,IAAI,OAAO,QAAQ;MACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;MAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;KAEJ;IACF;IAGA,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;KACA,eAAe;IACjB,CAAC;IACD,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;IACjE,MAAM,iBAAiB,MAAM,6BAC3B,IACA,oBACA,KAAK,WACL,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,KAAK,gBAAgB,CAChC;IAEA,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IAOA,OAAO;KAAC;KAAM;KAAkB;KAAgB,SALhC,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBAGyB;IAAO;GACzD,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GAKA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI;IAC7B,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,aACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAO9E,OAAO;IACL;IACA,SAPc,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aAKrB;IACA,aAAa,MAAM,6BACjB,IACA,oBACA,MACA,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,GAAG,CACd;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,MAAM,mBAAmB,IAAI;EAC7B,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,EAC9B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}
|
|
1
|
+
{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n PG_CONFIGURATION_LIMIT_EXCEEDED,\n PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n createReplicationSessionFor,\n endSession,\n extractConnectionConfig,\n keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n InitialSync,\n Restore,\n type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n createReplica,\n getReplicaState,\n getRestoreCandidates,\n metadataPublicationName,\n replicationSlotExpression,\n replicationSlotPrefix,\n type BackupOptions,\n type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n slot_name: string;\n consistent_point: string;\n snapshot_name: string;\n output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n slot: ReplicationSlot;\n capturedSnapshot: T;\n initialSession: Readable;\n replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n slot: string;\n reservation: Readable;\n replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n slotName: string;\n\n // Note: ignored if pgVersion < PG_17.\n failover?: boolean;\n\n // Create a temporary slot (i.e. not persisted, and automatically\n // cleaned up when the replication session ends).\n temporary?: boolean;\n\n // For overriding in tests.\n lockTimeout?: number;\n\n // A (non-replication) connection to the same upstream instance, used to\n // log what slot creation was waiting on if it times out.\n diagnosticsDB?: PostgresDB | undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n// so all commands must be sent using sql.unsafe(). This is technically safe\n// because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n lc: LogContext,\n session: postgres.Sql,\n {\n slotName,\n failover,\n temporary,\n lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n diagnosticsDB,\n }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n // does NOT apply to replication commands, but lock_timeout does (it governs\n // the heavyweight lock wait inside LockAcquire). Setting it here causes\n // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n // walsender, rather than relying solely on the client-side orTimeout\n // which can leave an orphaned backend.\n //\n // An orphaned walsender is actively harmful: by this point the replication\n // slot has already been created and is pinning WAL retention and catalog_xmin.\n // Worse, the slot is marked `active` (the walsender PID is still alive), so\n // the existing cleanup code (which drops inactive slots on retry) can't\n // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n // fires (~2h default) or the blocking transaction finishes.\n await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n const {pgVersion, pid: walsenderPID} = (\n await session.unsafe<{pgVersion: number; pid: number}[]>(`\n SELECT current_setting('server_version_num') as \"pgVersion\",\n pg_backend_pid() as \"pid\";\n `)\n )[0];\n\n const maybeTemporary = temporary ? 'TEMPORARY' : '';\n const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n if (raced === 'timed-out') {\n if (diagnosticsDB) {\n await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n }\n // Create slot can block indefinitely waiting for old transactions. End\n // this connection in the background and fail fast so the process restarts.\n void session\n .end()\n .catch(e =>\n lc.warn?.(`Error closing timed out replication slot session`, e),\n );\n throw new Error(\n `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n );\n }\n const [slot] = raced;\n lc.info?.(`Created replication slot ${slotName}`, slot);\n return slot;\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/**\n * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n *\n * Query text is deliberately omitted since it can contain row values.\n *\n * Exported for testing.\n */\nexport async function logSlotCreationBlockers(\n lc: LogContext,\n sql: PostgresDB,\n walsenderPID: number,\n slotName: string,\n) {\n try {\n const result = await orTimeout(\n Promise.all([\n sql<JSONObject[]> /*sql*/ `\n SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n pg_blocking_pids(pid) as \"blockingPIDs\"\n FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n sql<JSONObject[]> /*sql*/ `\n SELECT pid, datname, usename, application_name as \"applicationName\",\n backend_type as \"backendType\", state,\n wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n backend_xid::text as xid,\n xact_start::text as \"xactStart\",\n extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n FROM pg_stat_activity\n WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n ORDER BY xact_start NULLS LAST\n LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n ]),\n SLOT_DIAGNOSTICS_TIMEOUT_MS,\n );\n if (result === 'timed-out') {\n lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n return;\n }\n const [[walsender], transactions] = result;\n lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n walsender: walsender ?? null,\n transactions,\n });\n } catch (e) {\n lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n }\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n * replica slot management. This is the same lock that cleanup logic\n * acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n * replication session. The API of CREATE_REPLICATION_SLOT is such\n * that it cannot be done in a transaction, and cannot be followed by\n * any writes, or else its snapshot (which is needed for initial sync)\n * would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n * in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n * will not use the same name for the replication slot (which is selected\n * from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n * rank) will not delete the new slot during their cleanup logic, since\n * the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n lc: LogContext,\n sql: PostgresDB,\n sessionName: string,\n shard: ShardID,\n epoch: number,\n replicaID: string,\n failover: boolean,\n backupOptions: BackupOptions,\n captureSnapshot: (snapshot: string) => Promise<T>,\n stage: ReplicaStage,\n): Promise<ReplicationSlotResult<T>> {\n // Note: The replicationSession is used to create the replication slot\n // and closed immediately afterwards, or on error.\n const replicationSession = createReplicationSessionFor(sql, sessionName);\n const lockName = replicationSlotManagementLock(shard);\n const slotPoolPrefix = replicationSlotPrefix(shard);\n\n for (let first = true; ; first = false) {\n await dropUnclaimedSlots(lc, sql, shard);\n\n let slotName: string | undefined;\n let initialSession: Readable | undefined;\n try {\n return await runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n if (stage === InitialSync) {\n // With the lock acquired, ensure that only one initial sync is\n // active at a time. getRestoreCandidates() orders its results\n // with stage = InitialSync (and active = true) first.\n const others = await getRestoreCandidates(tx, shard, epoch);\n lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n if (others.length) {\n const [{id, stage, active}] = others;\n if (stage === InitialSync && active) {\n throw new AutoResetSignal(\n `another replica (${id}) is performing initial sync`,\n );\n }\n }\n }\n\n // Pick an available slotName from the slotPoolPrefix pool.\n const names = await tx<{name: string}[]> /*sql*/ `\n SELECT slot_name as name FROM pg_replication_slots\n WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n `.values();\n const inUse = new Set(names.flat());\n for (let next = 0; ; next++) {\n const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n if (!inUse.has(candidateName)) {\n slotName = candidateName;\n break;\n }\n }\n\n const slot = await createReplicationSlot(lc, replicationSession, {\n slotName,\n failover,\n diagnosticsDB: sql,\n });\n const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n await replicationSession.end();\n initialSession = await keepSlotActiveUntilTakenOver(lc, {\n db: extractConnectionConfig(sql),\n slot: slot.slot_name,\n dummyPublication: metadataPublicationName(\n shard.appID,\n shard.shardNum,\n ),\n lsn: String(toBigInt(slot.consistent_point)),\n });\n\n await createReplica(\n tx,\n shard,\n replicaID,\n slot.slot_name,\n epoch,\n stage === InitialSync\n ? toStateVersionString(slot.consistent_point)\n : '', // initialized with initRestoreReplica\n backupOptions,\n stage,\n );\n\n const replica = must(\n await getReplicaState(tx, shard, replicaID),\n `replica ${replicaID} was not created`,\n );\n\n return {slot, capturedSnapshot, initialSession, replica};\n });\n } catch (e) {\n if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n // Some Postgres variants (e.g. Google Cloud SQL) require that\n // the user have the REPLICATION role in order to create a slot.\n // Note that this must be done by the upstreamDB connection, and\n // does not work in the replicationSession itself.\n await sql`ALTER ROLE current_user WITH REPLICATION`;\n lc.info?.(`Added the REPLICATION role to database user`);\n continue;\n }\n // Note: This is currently manually tested since max_replication_slots\n // is a PG startup parameter that other tests depend on.\n // TODO: Figure out a way to unit test this (with the full PG setup).\n if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n lc.warn?.(\n `Reached max replication slots. Attempting to clean up unused slots`,\n e,\n );\n // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n await sql`\n DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n continue; // then let dropUnclaimedSlots() perform its cleanup\n }\n if (initialSession) {\n try {\n await endSession(initialSession);\n } catch {}\n }\n // Otherwise, clean up any created slot if something went wrong. The\n // slot may not exist (e.g. if its creation failed), and its name may\n // have been reused since the lock was released, so leave it to\n // dropUnclaimedSlots(), which only drops slots without a replica.\n if (slotName) {\n lc.warn?.(`deleting slot ${slotName} due to error`, e);\n try {\n await replicationSession.end();\n } catch {}\n await dropUnclaimedSlots(lc, sql, shard);\n }\n throw e;\n }\n }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n _sessionName: string,\n {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n const lockName = replicationSlotManagementLock(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n let reservation: Readable | undefined;\n try {\n const reserved = await runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n SELECT active, confirmed_flush_lsn as lsn\n FROM pg_replication_slots\n JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n `;\n if (replicas.length === 0) {\n lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n return null;\n }\n const [{active, lsn}] = replicas;\n if (active) {\n lc.warn?.(\n `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n );\n return null;\n }\n // Mark the replica as Restoring.\n await tx`\n UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n `;\n const replica = must(\n await getReplicaState(tx, shard, replicaID),\n `replica ${replicaID} disappeared`,\n );\n\n reservation = await keepSlotActiveUntilTakenOver(lc, {\n db: extractConnectionConfig(sql),\n slot,\n dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n lsn: String(toBigInt(lsn)),\n });\n return {\n slot,\n replica,\n reservation,\n };\n });\n lc.info?.(`successfully claimed slot ${slot}`);\n return reserved;\n } catch (e) {\n if (reservation) {\n try {\n await endSession(reservation);\n } catch {}\n }\n lc.error?.(`unable to claim replication slot ${slot}`, e);\n return null;\n }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n slots: string[],\n) {\n const lockName = replicationSlotManagementLock(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n return runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const dropped = await tx<{slot: string; replicaID: string | null}[]>\n /*sql*/ `\n SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name IN ${tx(slots)} AND NOT active;\n `;\n if (dropped.length) {\n lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n dropped,\n });\n }\n\n const replicas = dropped\n .map(({replicaID}) => replicaID)\n .filter(id => id !== null);\n if (replicas.length) {\n // Delete replicas associated with the inactive (and now dropped) slots.\n await tx\n /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n }\n });\n}\n\nfunction dropUnclaimedSlots(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n // The slot / replica cleanup happens within a transaction while holding\n // the replication slot management lock for this shard, to ensure that no\n // slot that belongs to a newer replica is dropped.\n const lockName = replicationSlotManagementLock(shard);\n const slotExpression = replicationSlotExpression(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n return runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const dropped = await tx /*sql*/ `\n SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name LIKE ${slotExpression} \n AND NOT active\n AND replica.id IS NULL;\n `;\n if (dropped.length) {\n lc.info?.(`dropped inactive replication slots`, {dropped});\n }\n\n // Conversely, delete replicas whose slots no longer exist, e.g. because\n // they were dropped outside of zero-cache's control (such as in an\n // upstream failover). Slot names are reused, so such rows would otherwise\n // be associated with a new slot of the same name. This is done (and\n // committed) before the slot is created, both so that the rows are never\n // associated with it, and because CREATE_REPLICATION_SLOT waits for\n // transactions that have written (i.e. been assigned an xid).\n const orphaned = await tx<{id: string}[]> /*sql*/ `\n DELETE FROM ${tx(replicasTable)} replica\n WHERE NOT EXISTS (\n SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n )\n RETURNING id;\n `;\n if (orphaned.length) {\n lc.warn?.(`deleted replicas whose slots no longer exist`, {\n replicas: orphaned.map(({id}) => id),\n });\n }\n\n const remaining = await tx<\n {slot: string; pid: number | null; id: string | null}[]\n > /*sql*/ `\n SELECT slot_name as slot, active_pid as pid, replica.id as id\n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name LIKE ${slotExpression};\n `;\n if (remaining.length) {\n lc.info?.(`remaining replication slots`, {remaining});\n }\n\n let active = 0;\n let draining = 0;\n for (const {id} of remaining) {\n if (id === null) {\n draining++;\n } else {\n active++;\n }\n }\n\n return {\n dropped: dropped.length,\n active,\n draining,\n };\n });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n n++; // Adjust for 0-based indexing\n\n let suffix = '';\n while (n > 0) {\n n--;\n suffix = ALPHABET[n % 26] + suffix;\n n = Math.floor(n / 26);\n }\n return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;AAqFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,GACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAIhE,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EACzB,IAAI,eACF,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAIzE,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACmC;CAGnC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAElD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAEvC,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAE1D,IAAI,UAAA,GAAuB;KAIzB,MAAM,SAAS,MAAM,qBAAqB,IAAI,OAAO,KAAK;KAC1D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;KAClE,IAAI,OAAO,QAAQ;MACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;MAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;KAEJ;IACF;IAGA,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;KACA,eAAe;IACjB,CAAC;IACD,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;IACjE,MAAM,mBAAmB,IAAI;IAC7B,iBAAiB,MAAM,6BAA6B,IAAI;KACtD,IAAI,wBAAwB,GAAG;KAC/B,MAAM,KAAK;KACX,kBAAkB,wBAChB,MAAM,OACN,MAAM,QACR;KACA,KAAK,OAAO,SAAS,KAAK,gBAAgB,CAAC;IAC7C,CAAC;IAED,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IAEA,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBACvB;IAEA,OAAO;KAAC;KAAM;KAAkB;KAAgB;IAAO;GACzD,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GACA,IAAI,gBACF,IAAI;IACF,MAAM,WAAW,cAAc;GACjC,QAAQ,CAAC;GAMX,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,IAAI;KACF,MAAM,mBAAmB,IAAI;IAC/B,QAAQ,CAAC;IACT,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,cACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;CACJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAE9E,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aACvB;GAEA,cAAc,MAAM,6BAA6B,IAAI;IACnD,IAAI,wBAAwB,GAAG;IAC/B;IACA,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,GAAG,CAAC;GAC3B,CAAC;GACD,OAAO;IACL;IACA;IACA;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aACF,IAAI;GACF,MAAM,WAAW,WAAW;EAC9B,QAAQ,CAAC;EAEX,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,EAC9B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}
|
|
@@ -13,7 +13,7 @@ import { discoverChangeStreamerAddress } from "./schema/tables.js";
|
|
|
13
13
|
import "./change-streamer.js";
|
|
14
14
|
import "./snapshot-message.js";
|
|
15
15
|
import { subscribeDownstreamSchema, subscribeUpstreamSchema } from "./subscribe.js";
|
|
16
|
-
import WebSocket from "ws";
|
|
16
|
+
import WebSocket$1 from "ws";
|
|
17
17
|
import websocket from "@fastify/websocket";
|
|
18
18
|
//#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
|
|
19
19
|
var MIN_SUPPORTED_PROTOCOL_VERSION = 4;
|
|
@@ -210,7 +210,7 @@ var ChangeStreamerHttpClient = class {
|
|
|
210
210
|
* replication-manager is the one that serves the subscription.
|
|
211
211
|
*/
|
|
212
212
|
async connect(taskID) {
|
|
213
|
-
const ws = new WebSocket(await this.#resolveChangeStreamer(SUBSCRIBE_PATH) + `?${new URLSearchParams({ taskID }).toString()}`, { handshakeTimeout: CONNECT_TIMEOUT_MS });
|
|
213
|
+
const ws = new WebSocket$1(await this.#resolveChangeStreamer(SUBSCRIBE_PATH) + `?${new URLSearchParams({ taskID }).toString()}`, { handshakeTimeout: CONNECT_TIMEOUT_MS });
|
|
214
214
|
const outbound = Subscription.create();
|
|
215
215
|
return {
|
|
216
216
|
instream: await streamInternalWithSize(this.#lc, ws, subscribeDownstreamSchema, outbound),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#changes","#reserveSnapshot","#subscribe","#receiveWebsocket","#ensureChangeStreamerStarted","#forwardReservation","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n readHead,\n streamInternal,\n streamInternalStringified,\n streamInternalWithSize,\n type Sized,\n type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n PROTOCOL_VERSION,\n type ChangeStreamer,\n type ChangeStreamerService,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n subscribeDownstreamSchema,\n subscribeUpstreamSchema,\n type ReservedMessage,\n type SnapshotStatus,\n type SubscribeContext,\n type SubscribeDownstream,\n type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\n// Max time for a client to establish the /subscribe websocket (TCP connect\n// plus upgrade) before abandoning the attempt.\nconst CONNECT_TIMEOUT_MS = 10_000;\n\ntype Options = HttpOptions & {\n startupDelayMs: number;\n config?:\n | Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>\n | undefined;\n getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n readonly id = 'change-streamer-http-server';\n readonly #lc: LogContext;\n readonly #opts: Options;\n readonly #changeStreamer: ChangeStreamerService;\n\n constructor(\n lc: LogContext,\n opts: Options,\n parent: Worker,\n changeStreamer: ChangeStreamerService,\n ) {\n super('change-streamer-http-server', lc, opts, async fastify => {\n await fastify.register(websocket);\n\n fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n fastify.get(\n SNAPSHOT_PATH_PATTERN,\n {websocket: true},\n this.#reserveSnapshot,\n );\n fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n fastify.get('/profz', (req, res) =>\n handleProfzRequest(\n lc,\n opts.config ?? {\n adminPassword: undefined,\n operatorPassword: undefined,\n },\n req,\n res,\n opts.getProfileWorker,\n undefined,\n 'change-streamer',\n ),\n );\n\n installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n lc,\n fastify.websocketServer,\n this.#receiveWebsocket,\n parent,\n );\n });\n\n this.#lc = lc;\n this.#opts = opts;\n this.#changeStreamer = changeStreamer;\n }\n\n // Called when receiving a web socket via the main dispatcher handoff.\n readonly #receiveWebsocket = (\n ws: WebSocket,\n action: 'changes' | 'snapshot' | 'subscribe',\n msg: IncomingMessageSubset,\n ) => {\n switch (action) {\n case 'snapshot':\n return this.#reserveSnapshot(ws, msg);\n case 'changes':\n return this.#changes(ws, msg);\n case 'subscribe':\n return this.#subscribe(ws, msg);\n default:\n closeWithError(\n this._lc,\n ws,\n `invalid action \"${action}\" received in handoff`,\n );\n return;\n }\n };\n\n readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n try {\n const url = new URL(\n req.url ?? '',\n req.headers.origin ?? 'http://localhost',\n );\n checkProtocolVersion(url.pathname);\n const taskID = url.searchParams.get('taskID');\n if (!taskID) {\n throw new Error('Missing taskID in snapshot request');\n }\n const downstream =\n await this.#changeStreamer.startSnapshotReservation(taskID);\n // Send-only for now: the reservation carries no inbound application\n // messages (only transport acks, demuxed internally).\n void streamInternal(this._lc, ws, undefined, downstream);\n } catch (err) {\n closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n }\n };\n\n readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n try {\n const ctx = getSubscriberContext(req);\n if (ctx.mode === 'serving') {\n this.#ensureChangeStreamerStarted('incoming subscription');\n }\n\n const downstream = await this.#changeStreamer.subscribe(ctx);\n // Send-only for now: the subscription carries no inbound application\n // messages (only transport acks, demuxed internally).\n void streamInternalStringified(this._lc, ws, undefined, downstream, {\n batched: ctx.wsBatched,\n });\n } catch (err) {\n closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n }\n };\n\n /**\n * The merged v7 `/subscribe` handler. A single bidirectional connection\n * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n * `reserved`) and then the subscription phase (`start-subscription` → change\n * stream). This ensures that the subscriber subscribes to the same task from\n * which it restored the backup.\n */\n readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n let reservation: Source<SnapshotMessage> | undefined;\n const outbound = Subscription.create<string | PreSerializedBatch>();\n\n try {\n const url = new URL(\n req.url ?? '',\n req.headers.origin ?? 'http://localhost',\n );\n const protocolVersion = checkProtocolVersion(url.pathname);\n if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n throw new Error(\n `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n `(client is at v${protocolVersion})`,\n );\n }\n\n const instream = await streamInternalStringified(\n this._lc,\n ws,\n subscribeUpstreamSchema,\n outbound,\n {batched: true},\n );\n\n // Keep reading upstream control messages for the life of the connection.\n // Do not manually exit loop, as that would close the socket. The loop\n // exits naturally when the connection closes.\n for await (const msg of instream) {\n if (msg[0] === 'reserve-snapshot') {\n const {taskID} = msg[1];\n this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n reservation =\n await this.#changeStreamer.startSnapshotReservation(taskID);\n void this.#forwardReservation(reservation, outbound);\n } else if (msg[0] === 'start-subscription') {\n const ctx: SubscriberContext = {\n ...msg[1],\n protocolVersion,\n wsBatched: true,\n };\n if (ctx.mode === 'serving') {\n this.#ensureChangeStreamerStarted('incoming subscription');\n }\n // subscribe() registers the subscriber and then releases any\n // reservation for that task.\n reservation = undefined;\n await this.#changeStreamer.subscribe(ctx, outbound);\n }\n }\n } catch (err) {\n closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n } finally {\n // Covers a reservation-phase disconnect (instream ended before a\n // subscription started); idempotent with the close handler above.\n reservation?.cancel();\n }\n };\n\n /**\n * Forwards a snapshot reservation's single `['status', ...]` message onto\n * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n * The loop then blocks on the held-open reservation until it is cancelled,\n * by `subscribe()` at the subscription handoff, or by the reservation-phase\n * teardown on disconnect.\n */\n async #forwardReservation(\n reservation: Source<SnapshotMessage>,\n outbound: Subscription<string | PreSerializedBatch>,\n ) {\n try {\n for await (const [, status] of reservation) {\n // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n outbound.push(\n BigIntJSON.stringify([\n 'reserved',\n {...status, tag: 'snapshot'},\n ] satisfies ReservedMessage),\n );\n }\n } catch (e) {\n this._lc.warn?.(`error forwarding snapshot reservation`, e);\n outbound.fail(e instanceof Error ? e : new Error(String(e)));\n }\n }\n\n #changeStreamerStarted = false;\n\n #ensureChangeStreamerStarted(reason?: string) {\n if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n this.#lc.info?.(\n `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n );\n void this.#changeStreamer\n .run()\n .catch(e =>\n this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n )\n .finally(() => this.stop());\n\n this.#changeStreamerStarted = true;\n }\n }\n\n protected override _onStart(): void {\n const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n if (startupDelayMs === 0) {\n // In RMv2, there is no need to delay starting the change streamer\n // because RM startup is non-disruptive.\n this.#ensureChangeStreamerStarted();\n } else {\n // In RMv1, starting the change-streamer forcibly shuts down the\n // previous change-streamer, causing view-syncers to reconnect.\n // If this replication-manager has just started, the routing layer may\n // not have registered it with DNS, as that only happens after it\n // confirms health checks. To minimize downtime, the takeover is\n // delayed for the configured startupDelayMs _after_ beginning to\n // advertise readiness.\n void readinessGate.then(() => {\n this.#lc.info?.(\n `waiting ${startupDelayMs}ms before taking over the change log`,\n );\n this._state.setTimeout(\n () =>\n this.#ensureChangeStreamerStarted(\n `startup delay elapsed (${startupDelayMs} ms)`,\n ),\n startupDelayMs,\n );\n });\n }\n }\n\n protected override async _onStop(): Promise<void> {\n if (this.#changeStreamerStarted) {\n await this.#changeStreamer.stop();\n }\n }\n}\n\nexport interface SnapshotReserver {\n reserveSnapshot(taskID: string): Promise<{\n reserved: SnapshotStatus;\n followup: ReservationFollowup;\n }>;\n}\n\nexport interface SubscriptionStarter {\n subscribe(\n ctx: SubscriberContext,\n ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n /**\n * A signal to indicate that the followup is no longer valid (i.e. the\n * connection to the reserving task has been severed).\n */\n readonly signal: AbortSignal;\n\n /**\n * Abandons the reservation without subscribing, closing the connection that\n * was held open to pin the change-log floor. Used when the caller cannot\n * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n * progress) and must not leave the reservation's connection open forever.\n */\n cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n readonly #lc: LogContext;\n readonly #shardID: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #changeStreamerURI: string | undefined;\n\n constructor(\n lc: LogContext,\n shardID: ShardID,\n changeDB: string,\n changeStreamerURI: string | undefined,\n ) {\n this.#lc = lc;\n this.#shardID = shardID;\n // Create a pg client with a single short-lived connection for the purpose\n // of change-streamer discovery (i.e. ChangeDB as DNS).\n this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n max: 1,\n ['idle_timeout']: 15,\n });\n this.#changeStreamerURI = changeStreamerURI;\n }\n\n async #resolveChangeStreamer(path: string) {\n let baseURL = this.#changeStreamerURI;\n if (!baseURL) {\n const address = await discoverChangeStreamerAddress(\n this.#shardID,\n this.#changeDB,\n );\n if (!address) {\n throw new Error(`no change-streamer is running`);\n }\n baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n }\n const uri = new URL(path, baseURL);\n this.#lc.info?.(`connecting to change-streamer@${uri}`);\n return uri;\n }\n\n async reserveSnapshot(taskID: string): Promise<{\n reserved: SnapshotStatus;\n followup: ReservationFollowup;\n }> {\n const {instream, outbound} = await this.connect(taskID);\n outbound.push(['reserve-snapshot', {taskID}]);\n\n const {head, rest} = readHead(instream);\n const item = await head.next();\n if (!item.value) {\n throw new Error('stream closed without snapshot reservation');\n }\n const {data: msg} = item.value;\n if (msg[0] !== 'reserved') {\n throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n }\n const reserved: SnapshotStatus = msg[1];\n const followup: ReservationFollowup = {\n subscribe: ctx => {\n outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n return Promise.resolve(rest);\n },\n signal: outbound.signal,\n cancel: reason => outbound.cancel(reason),\n };\n\n return {reserved, followup};\n }\n\n async subscribe(\n ctx: SubscriberContext,\n ): Promise<Source<Sized<SubscribeDownstream>>> {\n const {instream, outbound} = await this.connect(ctx.taskID);\n outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n return instream;\n }\n\n /**\n * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n * inbound change/reservation stream and an `outbound` sink into which the\n * caller pushes the `reserve-snapshot` / `start-subscription` control\n * messages that drive the connection's phases. The single connection spans\n * the reservation, backup restore, and subscription, so the reserving\n * replication-manager is the one that serves the subscription.\n */\n // exported for testing\n async connect(taskID: string): Promise<{\n instream: Source<Sized<SubscribeDownstream>>;\n outbound: Subscription<SubscribeUpstream>;\n }> {\n const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n // taskID is carried in the query for observability/routing; the server\n // reads the authoritative taskID from the control messages themselves.\n const params = new URLSearchParams({taskID});\n // Bound the TCP connect + HTTP upgrade. Without this, a SYN black-holed\n // by a stale Service endpoint (e.g. a just-terminated replication-manager\n // pod) is retransmitted by the kernel for ~2 minutes before ETIMEDOUT.\n // Failing fast lets the caller's retry open a fresh connection that is\n // routed to a live endpoint.\n const ws = new WebSocket(uri + `?${params.toString()}`, {\n handshakeTimeout: CONNECT_TIMEOUT_MS,\n });\n\n const outbound = Subscription.create<SubscribeUpstream>();\n const instream = await streamInternalWithSize(\n this.#lc,\n ws,\n subscribeDownstreamSchema,\n outbound,\n );\n return {instream, outbound};\n }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n const protocolVersion = checkProtocolVersion(url.pathname);\n const params = new URLParams(url);\n\n return {\n protocolVersion,\n id: params.get('id', true),\n taskID: params.get('taskID', true),\n mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n replicaVersion: params.get('replicaVersion', true),\n watermark: params.get('watermark', true),\n // Absent for subscribers that predate the parameter, which is the safe\n // default: the barrier falls back to polling rather than waiting on an\n // ACK that would never be attributed to a writer.\n wsBatched: params.getBoolean('wsBatched'),\n };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n const match = PATH_REGEX.exec(pathname);\n if (!match) {\n throw new Error(`invalid path: ${pathname}`);\n }\n const v = Number(match.groups?.version);\n if (\n Number.isNaN(v) ||\n v > PROTOCOL_VERSION ||\n v < MIN_SUPPORTED_PROTOCOL_VERSION\n ) {\n throw new Error(\n `Cannot service client at protocol v${v}. ` +\n `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n );\n }\n return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n const {taskID, id, mode, replicaVersion, watermark, backfills} = ctx;\n return {taskID, id, mode, replicaVersion, watermark, backfills};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAIvB,IAAM,qBAAqB;AAU3B,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAKC,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU;IACb,eAAe,KAAA;IACf,kBAAkB,KAAA;GACpB,GACA,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAKE,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKE,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKL,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,EAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,WAAU,IAAI;IACrB,KAAKK,6BAA6B,+BAA+B;IACjE,cACE,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;IAC5D,KAAUM,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAKD,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAKL,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAMM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAKC,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKR;EAC3D,IAAI,mBAAmB,GAGrB,KAAKM,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAKP,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKE,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AA+BA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAGnB;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CAAC,oBAAoB,EAAC,OAAM,CAAC,CAAC;EAE5C,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EAWD,MAAM,KAAK,IAAI,UAAU,MAVP,KAAKa,uBAAuB,cAAc,IAU7B,IAAI,IANhB,gBAAgB,EAAC,OAAM,CAMP,EAAO,SAAS,KAAK,EACtD,kBAAkB,mBACpB,CAAC;EAED,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAKb,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EAAC,QAAQ,IAAI,MAAM,gBAAgB,WAAW,cAAa;CACjE,OAAO;EAAC;EAAQ;EAAI;EAAM;EAAgB;EAAW;CAAS;AAChE"}
|
|
1
|
+
{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#changes","#reserveSnapshot","#subscribe","#receiveWebsocket","#ensureChangeStreamerStarted","#forwardReservation","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n readHead,\n streamInternal,\n streamInternalStringified,\n streamInternalWithSize,\n type Sized,\n type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n PROTOCOL_VERSION,\n type ChangeStreamer,\n type ChangeStreamerService,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n subscribeDownstreamSchema,\n subscribeUpstreamSchema,\n type ReservedMessage,\n type SnapshotStatus,\n type SubscribeContext,\n type SubscribeDownstream,\n type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\n// Max time for a client to establish the /subscribe websocket (TCP connect\n// plus upgrade) before abandoning the attempt.\nconst CONNECT_TIMEOUT_MS = 10_000;\n\ntype Options = HttpOptions & {\n startupDelayMs: number;\n config?:\n | Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>\n | undefined;\n getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n readonly id = 'change-streamer-http-server';\n readonly #lc: LogContext;\n readonly #opts: Options;\n readonly #changeStreamer: ChangeStreamerService;\n\n constructor(\n lc: LogContext,\n opts: Options,\n parent: Worker,\n changeStreamer: ChangeStreamerService,\n ) {\n super('change-streamer-http-server', lc, opts, async fastify => {\n await fastify.register(websocket);\n\n fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n fastify.get(\n SNAPSHOT_PATH_PATTERN,\n {websocket: true},\n this.#reserveSnapshot,\n );\n fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n fastify.get('/profz', (req, res) =>\n handleProfzRequest(\n lc,\n opts.config ?? {\n adminPassword: undefined,\n operatorPassword: undefined,\n },\n req,\n res,\n opts.getProfileWorker,\n undefined,\n 'change-streamer',\n ),\n );\n\n installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n lc,\n fastify.websocketServer,\n this.#receiveWebsocket,\n parent,\n );\n });\n\n this.#lc = lc;\n this.#opts = opts;\n this.#changeStreamer = changeStreamer;\n }\n\n // Called when receiving a web socket via the main dispatcher handoff.\n readonly #receiveWebsocket = (\n ws: WebSocket,\n action: 'changes' | 'snapshot' | 'subscribe',\n msg: IncomingMessageSubset,\n ) => {\n switch (action) {\n case 'snapshot':\n return this.#reserveSnapshot(ws, msg);\n case 'changes':\n return this.#changes(ws, msg);\n case 'subscribe':\n return this.#subscribe(ws, msg);\n default:\n closeWithError(\n this._lc,\n ws,\n `invalid action \"${action}\" received in handoff`,\n );\n return;\n }\n };\n\n readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n try {\n const url = new URL(\n req.url ?? '',\n req.headers.origin ?? 'http://localhost',\n );\n checkProtocolVersion(url.pathname);\n const taskID = url.searchParams.get('taskID');\n if (!taskID) {\n throw new Error('Missing taskID in snapshot request');\n }\n const downstream =\n await this.#changeStreamer.startSnapshotReservation(taskID);\n // Send-only for now: the reservation carries no inbound application\n // messages (only transport acks, demuxed internally).\n void streamInternal(this._lc, ws, undefined, downstream);\n } catch (err) {\n closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n }\n };\n\n readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n try {\n const ctx = getSubscriberContext(req);\n if (ctx.mode === 'serving') {\n this.#ensureChangeStreamerStarted('incoming subscription');\n }\n\n const downstream = await this.#changeStreamer.subscribe(ctx);\n // Send-only for now: the subscription carries no inbound application\n // messages (only transport acks, demuxed internally).\n void streamInternalStringified(this._lc, ws, undefined, downstream, {\n batched: ctx.wsBatched,\n });\n } catch (err) {\n closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n }\n };\n\n /**\n * The merged v7 `/subscribe` handler. A single bidirectional connection\n * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n * `reserved`) and then the subscription phase (`start-subscription` → change\n * stream). This ensures that the subscriber subscribes to the same task from\n * which it restored the backup.\n */\n readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n let reservation: Source<SnapshotMessage> | undefined;\n const outbound = Subscription.create<string | PreSerializedBatch>();\n\n try {\n const url = new URL(\n req.url ?? '',\n req.headers.origin ?? 'http://localhost',\n );\n const protocolVersion = checkProtocolVersion(url.pathname);\n if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n throw new Error(\n `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n `(client is at v${protocolVersion})`,\n );\n }\n\n const instream = await streamInternalStringified(\n this._lc,\n ws,\n subscribeUpstreamSchema,\n outbound,\n {batched: true},\n );\n\n // Keep reading upstream control messages for the life of the connection.\n // Do not manually exit loop, as that would close the socket. The loop\n // exits naturally when the connection closes.\n for await (const msg of instream) {\n if (msg[0] === 'reserve-snapshot') {\n const {taskID} = msg[1];\n this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n reservation =\n await this.#changeStreamer.startSnapshotReservation(taskID);\n void this.#forwardReservation(reservation, outbound);\n } else if (msg[0] === 'start-subscription') {\n const ctx: SubscriberContext = {\n ...msg[1],\n protocolVersion,\n wsBatched: true,\n };\n if (ctx.mode === 'serving') {\n this.#ensureChangeStreamerStarted('incoming subscription');\n }\n // subscribe() registers the subscriber and then releases any\n // reservation for that task.\n reservation = undefined;\n await this.#changeStreamer.subscribe(ctx, outbound);\n }\n }\n } catch (err) {\n closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n } finally {\n // Covers a reservation-phase disconnect (instream ended before a\n // subscription started); idempotent with the close handler above.\n reservation?.cancel();\n }\n };\n\n /**\n * Forwards a snapshot reservation's single `['status', ...]` message onto\n * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n * The loop then blocks on the held-open reservation until it is cancelled,\n * by `subscribe()` at the subscription handoff, or by the reservation-phase\n * teardown on disconnect.\n */\n async #forwardReservation(\n reservation: Source<SnapshotMessage>,\n outbound: Subscription<string | PreSerializedBatch>,\n ) {\n try {\n for await (const [, status] of reservation) {\n // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n outbound.push(\n BigIntJSON.stringify([\n 'reserved',\n {...status, tag: 'snapshot'},\n ] satisfies ReservedMessage),\n );\n }\n } catch (e) {\n this._lc.warn?.(`error forwarding snapshot reservation`, e);\n outbound.fail(e instanceof Error ? e : new Error(String(e)));\n }\n }\n\n #changeStreamerStarted = false;\n\n #ensureChangeStreamerStarted(reason?: string) {\n if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n this.#lc.info?.(\n `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n );\n void this.#changeStreamer\n .run()\n .catch(e =>\n this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n )\n .finally(() => this.stop());\n\n this.#changeStreamerStarted = true;\n }\n }\n\n protected override _onStart(): void {\n const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n if (startupDelayMs === 0) {\n // In RMv2, there is no need to delay starting the change streamer\n // because RM startup is non-disruptive.\n this.#ensureChangeStreamerStarted();\n } else {\n // In RMv1, starting the change-streamer forcibly shuts down the\n // previous change-streamer, causing view-syncers to reconnect.\n // If this replication-manager has just started, the routing layer may\n // not have registered it with DNS, as that only happens after it\n // confirms health checks. To minimize downtime, the takeover is\n // delayed for the configured startupDelayMs _after_ beginning to\n // advertise readiness.\n void readinessGate.then(() => {\n this.#lc.info?.(\n `waiting ${startupDelayMs}ms before taking over the change log`,\n );\n this._state.setTimeout(\n () =>\n this.#ensureChangeStreamerStarted(\n `startup delay elapsed (${startupDelayMs} ms)`,\n ),\n startupDelayMs,\n );\n });\n }\n }\n\n protected override async _onStop(): Promise<void> {\n if (this.#changeStreamerStarted) {\n await this.#changeStreamer.stop();\n }\n }\n}\n\nexport interface SnapshotReserver {\n reserveSnapshot(taskID: string): Promise<{\n reserved: SnapshotStatus;\n followup: ReservationFollowup;\n }>;\n}\n\nexport interface SubscriptionStarter {\n subscribe(\n ctx: SubscriberContext,\n ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n /**\n * A signal to indicate that the followup is no longer valid (i.e. the\n * connection to the reserving task has been severed).\n */\n readonly signal: AbortSignal;\n\n /**\n * Abandons the reservation without subscribing, closing the connection that\n * was held open to pin the change-log floor. Used when the caller cannot\n * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n * progress) and must not leave the reservation's connection open forever.\n */\n cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n readonly #lc: LogContext;\n readonly #shardID: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #changeStreamerURI: string | undefined;\n\n constructor(\n lc: LogContext,\n shardID: ShardID,\n changeDB: string,\n changeStreamerURI: string | undefined,\n ) {\n this.#lc = lc;\n this.#shardID = shardID;\n // Create a pg client with a single short-lived connection for the purpose\n // of change-streamer discovery (i.e. ChangeDB as DNS).\n this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n max: 1,\n ['idle_timeout']: 15,\n });\n this.#changeStreamerURI = changeStreamerURI;\n }\n\n async #resolveChangeStreamer(path: string) {\n let baseURL = this.#changeStreamerURI;\n if (!baseURL) {\n const address = await discoverChangeStreamerAddress(\n this.#shardID,\n this.#changeDB,\n );\n if (!address) {\n throw new Error(`no change-streamer is running`);\n }\n baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n }\n const uri = new URL(path, baseURL);\n this.#lc.info?.(`connecting to change-streamer@${uri}`);\n return uri;\n }\n\n async reserveSnapshot(taskID: string): Promise<{\n reserved: SnapshotStatus;\n followup: ReservationFollowup;\n }> {\n const {instream, outbound} = await this.connect(taskID);\n outbound.push(['reserve-snapshot', {taskID}]);\n\n const {head, rest} = readHead(instream);\n const item = await head.next();\n if (!item.value) {\n throw new Error('stream closed without snapshot reservation');\n }\n const {data: msg} = item.value;\n if (msg[0] !== 'reserved') {\n throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n }\n const reserved: SnapshotStatus = msg[1];\n const followup: ReservationFollowup = {\n subscribe: ctx => {\n outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n return Promise.resolve(rest);\n },\n signal: outbound.signal,\n cancel: reason => outbound.cancel(reason),\n };\n\n return {reserved, followup};\n }\n\n async subscribe(\n ctx: SubscriberContext,\n ): Promise<Source<Sized<SubscribeDownstream>>> {\n const {instream, outbound} = await this.connect(ctx.taskID);\n outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n return instream;\n }\n\n /**\n * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n * inbound change/reservation stream and an `outbound` sink into which the\n * caller pushes the `reserve-snapshot` / `start-subscription` control\n * messages that drive the connection's phases. The single connection spans\n * the reservation, backup restore, and subscription, so the reserving\n * replication-manager is the one that serves the subscription.\n */\n // exported for testing\n async connect(taskID: string): Promise<{\n instream: Source<Sized<SubscribeDownstream>>;\n outbound: Subscription<SubscribeUpstream>;\n }> {\n const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n // taskID is carried in the query for observability/routing; the server\n // reads the authoritative taskID from the control messages themselves.\n const params = new URLSearchParams({taskID});\n // Bound the TCP connect + HTTP upgrade. Without this, a SYN black-holed\n // by a stale Service endpoint (e.g. a just-terminated replication-manager\n // pod) is retransmitted by the kernel for ~2 minutes before ETIMEDOUT.\n // Failing fast lets the caller's retry open a fresh connection that is\n // routed to a live endpoint.\n const ws = new WebSocket(uri + `?${params.toString()}`, {\n handshakeTimeout: CONNECT_TIMEOUT_MS,\n });\n\n const outbound = Subscription.create<SubscribeUpstream>();\n const instream = await streamInternalWithSize(\n this.#lc,\n ws,\n subscribeDownstreamSchema,\n outbound,\n );\n return {instream, outbound};\n }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n const protocolVersion = checkProtocolVersion(url.pathname);\n const params = new URLParams(url);\n\n return {\n protocolVersion,\n id: params.get('id', true),\n taskID: params.get('taskID', true),\n mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n replicaVersion: params.get('replicaVersion', true),\n watermark: params.get('watermark', true),\n // Absent for subscribers that predate the parameter, which is the safe\n // default: the barrier falls back to polling rather than waiting on an\n // ACK that would never be attributed to a writer.\n wsBatched: params.getBoolean('wsBatched'),\n };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n const match = PATH_REGEX.exec(pathname);\n if (!match) {\n throw new Error(`invalid path: ${pathname}`);\n }\n const v = Number(match.groups?.version);\n if (\n Number.isNaN(v) ||\n v > PROTOCOL_VERSION ||\n v < MIN_SUPPORTED_PROTOCOL_VERSION\n ) {\n throw new Error(\n `Cannot service client at protocol v${v}. ` +\n `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n );\n }\n return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n const {taskID, id, mode, replicaVersion, watermark, backfills} = ctx;\n return {taskID, id, mode, replicaVersion, watermark, backfills};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAIvB,IAAM,qBAAqB;AAU3B,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAKC,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU;IACb,eAAe,KAAA;IACf,kBAAkB,KAAA;GACpB,GACA,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAKE,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKE,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKL,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,EAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,WAAU,IAAI;IACrB,KAAKK,6BAA6B,+BAA+B;IACjE,cACE,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;IAC5D,KAAUM,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAKD,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAKL,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAMM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAKC,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKR;EAC3D,IAAI,mBAAmB,GAGrB,KAAKM,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAKP,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKE,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AA+BA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAGnB;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CAAC,oBAAoB,EAAC,OAAM,CAAC,CAAC;EAE5C,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EAWD,MAAM,KAAK,IAAI,YAAU,MAVP,KAAKa,uBAAuB,cAAc,IAU7B,IAAI,IANhB,gBAAgB,EAAC,OAAM,CAMP,EAAO,SAAS,KAAK,EACtD,kBAAkB,mBACpB,CAAC;EAED,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAKb,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EAAC,QAAQ,IAAI,MAAM,gBAAgB,WAAW,cAAa;CACjE,OAAO;EAAC;EAAQ;EAAI;EAAM;EAAgB;EAAW;CAAS;AAChE"}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import type { LogContext } from '@rocicorp/logger';
|
|
2
2
|
import type { InitConnectionBody } from '../../../../zero-protocol/src/connect.ts';
|
|
3
|
+
import type { ProtocolFlags } from '../../../../zero-protocol/src/protocol-flag.ts';
|
|
3
4
|
import type { UpdateAuthBody } from '../../../../zero-protocol/src/update-auth.ts';
|
|
4
5
|
import { type Auth, type ValidateLegacyJWT } from '../../auth/auth.ts';
|
|
5
6
|
import type { ZeroConfig } from '../../config/zero-config.ts';
|
|
6
|
-
import type { ConnectParams
|
|
7
|
+
import type { ConnectParams } from '../../workers/connect-params.ts';
|
|
7
8
|
export type ConnectionState = 'provisional' | 'validated';
|
|
8
9
|
/**
|
|
9
10
|
* Normalized user identity shared by live connection state and group auth state.
|
|
@@ -59,7 +60,7 @@ export type ConnectionContext = {
|
|
|
59
60
|
readonly profileID: string | null;
|
|
60
61
|
readonly baseCookie: string | null;
|
|
61
62
|
readonly protocolVersion: number;
|
|
62
|
-
readonly
|
|
63
|
+
readonly protocolFlags: ProtocolFlags;
|
|
63
64
|
readonly revision: number;
|
|
64
65
|
readonly revalidateAt: number | undefined;
|
|
65
66
|
readonly insertionOrder: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connection-context-manager.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/connection-context-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,0CAA0C,CAAC;AAGjF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,8CAA8C,CAAC;AACjF,OAAO,EAGL,KAAK,IAAI,EACT,KAAK,iBAAiB,EACvB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAG5D,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"connection-context-manager.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/connection-context-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,0CAA0C,CAAC;AAGjF,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,gDAAgD,CAAC;AAClF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,8CAA8C,CAAC;AACjF,OAAO,EAGL,KAAK,IAAI,EACT,KAAK,iBAAiB,EACvB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAG5D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,iCAAiC,CAAC;AAEnE,MAAM,MAAM,eAAe,GAAG,aAAa,GAAG,WAAW,CAAC;AAE1D;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAA;CAAC,CAAC;AAErD;;;;GAIG;AACH,MAAM,MAAM,oBAAoB,GAC5B;IAAC,IAAI,EAAE,iBAAiB,CAAA;CAAC,GACzB;IAAC,IAAI,EAAE,kBAAkB,CAAC;IAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;CAAC,CAAC;AAE/D;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,KAAK,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;AAEvC,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,0DAA0D;IAC1D,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;IACtE,0DAA0D;IAC1D,QAAQ,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;IACvE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,QAAQ,CAAC,kBAAkB,EAAE,SAAS,UAAU,EAAE,GAAG,SAAS,CAAC;IAC/D,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;CACvC,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAEhC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAEzB,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;IAEhC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IAEtC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IAE1C,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAEhC,QAAQ,CAAC,YAAY,EAAE,sBAAsB,CAAC;IAC9C,QAAQ,CAAC,aAAa,EAAE,sBAAsB,CAAC;CAChD,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,CAAC,UAAU,EAAE,SAAS,GAAG,SAAS,CAAC;IAE3C,QAAQ,CAAC,oBAAoB,EAAE,kBAAkB,GAAG,SAAS,CAAC;IAC9D,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC;IAE3C,QAAQ,CAAC,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;CACrD,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,aAAa,EAAE,aAAa,EAC5B,IAAI,CAAC,EAAE,IAAI,GACV,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE/B,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,kBAAkB,GACvB,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE/B,UAAU,CACR,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAExC,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,oBAAoB,GAE9B,QAAQ,CAAC;QACP,UAAU,EAAE,iBAAiB,CAAC;QAC9B,KAAK,EAAE,cAAc,CAAC;KACvB,CAAC,GACF,SAAS,CAAC;IAEd,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAC3C,eAAe,CACb,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAE3C,gCAAgC,CAC9B,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,IAAI,CAAC;IAER,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;IAEhD,gBAAgB,CAAC,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,IAAI,CAAC;IAE3D,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAC3C,wBAAwB,CACtB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE/B,8BAA8B,IAAI,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAC1E,kCAAkC,IAAI,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAElE,aAAa,IAAI,QAAQ,CAAC,cAAc,CAAC,CAAC;IAE1C,eAAe,IAAI;QACjB,gBAAgB,EAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAChD,cAAc,EAAE,OAAO,CAAC;QACxB,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;KACxC,CAAC;CACH,CAAC;AAEF;;;;;;;GAOG;AACH,qBAAa,4BAA6B,YAAW,wBAAwB;;IAsB3E,YACE,EAAE,EAAE,UAAU,EACd,yBAAyB,CAAC,EAAE,MAAM,EAClC,0BAA0B,CAAC,EAAE,MAAM,EACnC,WAAW,CAAC,EAAE,WAAW,EACzB,UAAU,CAAC,EAAE,WAAW,EACxB,iBAAiB,CAAC,EAAE,iBAAiB,EACrC,GAAG,CAAC,EAAE,MAAM,MAAM,EAenB;IAED;;;;;OAKG;IACH,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,aAAa,EAAE,aAAa,EAC5B,IAAI,CAAC,EAAE,IAAI,GACV,QAAQ,CAAC,iBAAiB,CAAC,CA+C7B;IAED;;;;;OAKG;IACH,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,kBAAkB,GACvB,QAAQ,CAAC,iBAAiB,CAAC,CAiD7B;IAED;;;OAGG;IACG,UAAU,CACd,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CA4BtC;IAED;;;;;;;;OAQG;IACH,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,oBAAoB,GAE9B,QAAQ,CAAC;QACP,UAAU,EAAE,iBAAiB,CAAC;QAC9B,KAAK,EAAE,cAAc,CAAC;KACvB,CAAC,GACF,SAAS,CA6EZ;IAED,mGAAmG;IACnG,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,iBAAiB,GAAG,SAAS,CAE/B;IAED,6FAA6F;IAC7F,eAAe,CAAC,QAAQ,EAAE,kBAAkB,GAAG,iBAAiB,GAAG,SAAS,CAE3E;IAED;;;;OAIG;IACH,gCAAgC,CAC9B,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,IAAI,CAaN;IAED,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAM9C;IAED,gBAAgB,CAAC,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,IAAI,CAezD;IAED,iEAAiE;IACjE,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAEzC;IAED,gFAAgF;IAChF,wBAAwB,CACtB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,CAE7B;IAED,gEAAgE;IAChE,8BAA8B,IAAI,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAExE;IAED,kCAAkC,IAAI,QAAQ,CAAC,iBAAiB,CAAC,CAchE;IAED,2CAA2C;IAC3C,aAAa,IAAI,QAAQ,CAAC,cAAc,CAAC,CAExC;IAED;;;;;;;OAOG;IACH,eAAe,IAAI;QACjB,gBAAgB,EAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAChD,cAAc,EAAE,OAAO,CAAC;QACxB,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;KACxC,CA8CA;CAiNF"}
|
|
@@ -70,7 +70,7 @@ var ConnectionContextManagerImpl = class {
|
|
|
70
70
|
profileID: connectParams.profileID,
|
|
71
71
|
baseCookie: connectParams.baseCookie,
|
|
72
72
|
protocolVersion: connectParams.protocolVersion,
|
|
73
|
-
|
|
73
|
+
protocolFlags: connectParams.protocolFlags,
|
|
74
74
|
revalidateAt: void 0,
|
|
75
75
|
queryContext: getContext("query"),
|
|
76
76
|
mutateContext: getContext("mutate"),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connection-context-manager.js","names":["#lc","#connections","#validateLegacyJWT","#now","#revalidateIntervalMs","#retransformIntervalMs","#queryConfig","#pushConfig","#removeConnection","#nextInsertionOrder","#storeConnection","#refreshBackgroundConnectionContext","#updateBackgroundRetransformDeadline","#mustGetConnectionContext","#demoteConnection","#getConnectionContext","#group","#setGroup","#nextRevalidateAt","#getBackgroundConnectionContext","#sharedRetransformReady","#setBackgroundConnection"],"sources":["../../../../../../zero-cache/src/services/view-syncer/connection-context-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {InitConnectionBody} from '../../../../zero-protocol/src/connect.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport type {UpdateAuthBody} from '../../../../zero-protocol/src/update-auth.ts';\nimport {\n authEquals,\n resolveAuth,\n type Auth,\n type ValidateLegacyJWT,\n} from '../../auth/auth.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {compileUrlPattern} from '../../custom/fetch.ts';\nimport {ProtocolErrorWithLevel} from '../../types/error-with-level.ts';\nimport type {\n ConnectParams,\n FeatureFlagSet,\n} from '../../workers/connect-params.ts';\n\nexport type ConnectionState = 'provisional' | 'validated';\n\n/**\n * Normalized user identity shared by live connection state and group auth state.\n * `id: null` means logged out.\n */\nexport type UserState = {readonly id: string | null};\n\n/**\n * Delineates the two paths for validating a connection: either server can validate\n * the user's identity and return a definitive userID to trust, or we fall back to\n * trusting the one provided by the client in the incoming query params.\n */\nexport type ConnectionValidation =\n | {kind: 'client-fallback'}\n | {kind: 'server-validated'; validatedUserID: string | null};\n\n/**\n * Identifies one live websocket for a client slot.\n */\nexport type ConnectionSelector = {\n readonly clientID: string;\n readonly wsID: string;\n};\n\ntype FetchConfig = ZeroConfig['query'];\n\nexport type HeaderOptions = {\n readonly apiKey?: string | undefined;\n /** Allowlisted headers provided in the client options. */\n readonly customHeaders?: Readonly<Record<string, string>> | undefined;\n /** Allowlisted headers from the incoming HTTP request. */\n readonly requestHeaders?: Readonly<Record<string, string>> | undefined;\n readonly cookie?: string | undefined;\n readonly origin?: string | undefined;\n};\n\nexport type ConnectionFetchContext = {\n readonly url: string | undefined;\n readonly allowedUrlPatterns: readonly URLPattern[] | undefined;\n readonly headerOptions: HeaderOptions;\n};\n\n/**\n * A snapshot of one live connection tracked by the manager.\n *\n * `revalidateAt` is only populated while the connection is `validated`.\n */\nexport type ConnectionContext = {\n readonly state: ConnectionState;\n\n readonly clientID: string;\n readonly wsID: string;\n readonly user: UserState;\n\n readonly auth: Auth | undefined;\n\n readonly profileID: string | null;\n readonly baseCookie: string | null;\n readonly protocolVersion: number;\n readonly features: FeatureFlagSet;\n\n readonly revision: number;\n\n readonly revalidateAt: number | undefined;\n\n readonly insertionOrder: number;\n\n readonly queryContext: ConnectionFetchContext;\n readonly mutateContext: ConnectionFetchContext;\n};\n\n/**\n * Group-scoped auth state shared across the live connections.\n *\n * The background connection is the validated connection currently used for\n * shared background work. Retransform happens on a group level, and uses\n * the background connection's credential to refetch the latest queries.\n */\nexport type GroupAuthState = {\n readonly pinnedUser: UserState | undefined;\n\n readonly backgroundConnection: ConnectionSelector | undefined;\n readonly retransformAt: number | undefined;\n // Defer all maintenance in case a transient failure occurs.\n readonly maintenanceNotBeforeAt: number | undefined;\n};\n\nexport type ConnectionContextManager = {\n registerConnection(\n selector: ConnectionSelector,\n connectParams: ConnectParams,\n auth?: Auth,\n ): Readonly<ConnectionContext>;\n\n initConnection(\n selector: ConnectionSelector,\n body: InitConnectionBody,\n ): Readonly<ConnectionContext>;\n\n updateAuth(\n selector: ConnectionSelector,\n body: UpdateAuthBody,\n ): Promise<Readonly<ConnectionContext>>;\n\n validateConnection(\n selector: ConnectionSelector,\n revision: number,\n validation: ConnectionValidation,\n ):\n | Readonly<{\n connection: ConnectionContext;\n group: GroupAuthState;\n }>\n | undefined;\n\n failConnection(\n selector: ConnectionSelector,\n revision: number,\n ): Readonly<ConnectionContext> | undefined;\n closeConnection(\n selector: ConnectionSelector,\n ): Readonly<ConnectionContext> | undefined;\n\n markBackgroundRetransformSuccess(\n selector: ConnectionSelector,\n revision: number,\n ): void;\n\n setSharedRetransformReady(ready: boolean): void;\n\n deferMaintenance(kind: 'revalidate' | 'retransform'): void;\n\n getConnectionContext(\n selector: ConnectionSelector,\n ): Readonly<ConnectionContext> | undefined;\n mustGetConnectionContext(\n selector: ConnectionSelector,\n ): Readonly<ConnectionContext>;\n\n getBackgroundConnectionContext(): Readonly<ConnectionContext> | undefined;\n mustGetBackgroundConnectionContext(): Readonly<ConnectionContext>;\n\n getGroupState(): Readonly<GroupAuthState>;\n\n planMaintenance(): {\n dueRevalidations: Readonly<ConnectionContext>[];\n dueRetransform: boolean;\n earliestDeadlineAt: number | undefined;\n };\n};\n\n/**\n * State machine for the auth state of a single `ViewSyncerService`.\n *\n * Connections are registered as `provisional`, optionally backfilled with\n * `initConnection` metadata, and then promoted to `validated` once their\n * effective `userID` is confirmed as valid. The manager also tracks which\n * validated connection currently serves as the group's background connection.\n */\nexport class ConnectionContextManagerImpl implements ConnectionContextManager {\n readonly #lc: LogContext;\n\n // The live connection records, keyed by clientID\n readonly #connections = new Map<string, ConnectionContext>();\n #group: GroupAuthState = {\n pinnedUser: undefined,\n backgroundConnection: undefined,\n retransformAt: undefined,\n maintenanceNotBeforeAt: undefined,\n };\n\n readonly #validateLegacyJWT: ValidateLegacyJWT | undefined;\n\n readonly #now: () => number;\n readonly #revalidateIntervalMs: number | undefined;\n readonly #retransformIntervalMs: number | undefined;\n readonly #queryConfig: FetchConfig | undefined;\n readonly #pushConfig: FetchConfig | undefined;\n #sharedRetransformReady = false;\n #nextInsertionOrder = 0;\n\n constructor(\n lc: LogContext,\n revalidateIntervalSeconds?: number,\n retransformIntervalSeconds?: number,\n queryConfig?: FetchConfig,\n pushConfig?: FetchConfig,\n validateLegacyJWT?: ValidateLegacyJWT,\n now?: () => number,\n ) {\n this.#lc = lc;\n this.#now = now ?? Date.now;\n this.#revalidateIntervalMs =\n revalidateIntervalSeconds === undefined\n ? undefined\n : revalidateIntervalSeconds * 1000;\n this.#retransformIntervalMs =\n retransformIntervalSeconds === undefined\n ? undefined\n : retransformIntervalSeconds * 1000;\n this.#queryConfig = queryConfig;\n this.#pushConfig = pushConfig;\n this.#validateLegacyJWT = validateLegacyJWT;\n }\n\n /**\n * Creates or replaces the live record for a websocket connection.\n *\n * Re-registering the same `clientID` drops the old socket record and starts\n * the replacement back in `provisional` state.\n */\n registerConnection(\n selector: ConnectionSelector,\n connectParams: ConnectParams,\n auth?: Auth,\n ): Readonly<ConnectionContext> {\n this.#removeConnection(selector);\n\n const getContext = (type: 'query' | 'mutate'): ConnectionFetchContext => {\n const config = type === 'query' ? this.#queryConfig : this.#pushConfig;\n\n return {\n url: config?.url?.[0],\n allowedUrlPatterns: config?.url?.map(compileUrlPattern),\n headerOptions: {\n customHeaders: undefined,\n requestHeaders: filterHeaders(\n connectParams.requestHeaders,\n config?.allowedRequestHeaders,\n ),\n origin: connectParams.origin,\n apiKey: config?.apiKey,\n cookie: config?.forwardCookies ? connectParams.httpCookie : undefined,\n },\n };\n };\n\n const connection: ConnectionContext = {\n state: 'provisional',\n\n clientID: connectParams.clientID,\n wsID: connectParams.wsID,\n revision: 0,\n user: {id: connectParams.userID ?? null},\n auth,\n\n profileID: connectParams.profileID,\n baseCookie: connectParams.baseCookie,\n protocolVersion: connectParams.protocolVersion,\n features: connectParams.features,\n\n revalidateAt: undefined,\n\n queryContext: getContext('query'),\n mutateContext: getContext('mutate'),\n\n insertionOrder: ++this.#nextInsertionOrder,\n };\n this.#storeConnection(connection);\n this.#refreshBackgroundConnectionContext();\n this.#updateBackgroundRetransformDeadline(false);\n return connection;\n }\n\n /**\n * Backfills `initConnection` data for sockets that were registered before the\n * client could send its full init payload.\n *\n * This updates metadata only; it does not validate the connection.\n */\n initConnection(\n selector: ConnectionSelector,\n body: InitConnectionBody,\n ): Readonly<ConnectionContext> {\n const connection = this.#mustGetConnectionContext(selector);\n\n let queryContext = connection.queryContext;\n let mutateContext = connection.mutateContext;\n\n if (body.userQueryURL) {\n queryContext = {\n ...queryContext,\n url: body.userQueryURL,\n };\n }\n if (body.userQueryHeaders) {\n queryContext = {\n ...queryContext,\n headerOptions: {\n ...queryContext.headerOptions,\n customHeaders: filterHeaders(\n body.userQueryHeaders,\n this.#queryConfig?.allowedClientHeaders,\n ),\n },\n };\n }\n if (body.userPushURL) {\n mutateContext = {\n ...mutateContext,\n url: body.userPushURL,\n };\n }\n if (body.userPushHeaders) {\n mutateContext = {\n ...mutateContext,\n headerOptions: {\n ...mutateContext.headerOptions,\n customHeaders: filterHeaders(\n body.userPushHeaders,\n this.#pushConfig?.allowedClientHeaders,\n ),\n },\n };\n }\n\n return this.#demoteConnection({\n ...connection,\n revision: connection.revision + 1,\n queryContext,\n mutateContext,\n });\n }\n\n /**\n * A material auth change demotes the connection back to provisional until it\n * is validated again.\n */\n async updateAuth(\n selector: ConnectionSelector,\n body: UpdateAuthBody,\n ): Promise<Readonly<ConnectionContext>> {\n const connection = this.#mustGetConnectionContext(selector);\n\n const nextAuth = await resolveAuth(\n this.#lc,\n connection.auth,\n connection.user.id,\n body.auth,\n this.#validateLegacyJWT,\n );\n\n const authChanged = !authEquals(connection.auth, nextAuth);\n if (authChanged) {\n return this.#demoteConnection({\n ...connection,\n auth: nextAuth,\n revision: connection.revision + 1,\n });\n }\n\n if (nextAuth === connection.auth) {\n return connection;\n }\n\n return this.#storeConnection({\n ...connection,\n auth: nextAuth,\n });\n }\n\n /**\n * Validates one connection against the group's pinned `userID`.\n *\n * The first successful validation binds the group `userID`. Later\n * validations must match it. Validation also refreshes the connection's\n * revalidation deadline and may pick the connection as the group\n * background connection if none is currently available. If the websocket is\n * gone by the time async validation finishes, this becomes a no-op.\n */\n validateConnection(\n selector: ConnectionSelector,\n revision: number,\n validation: ConnectionValidation,\n ):\n | Readonly<{\n connection: ConnectionContext;\n group: GroupAuthState;\n }>\n | undefined {\n const connection = this.#getConnectionContext(selector);\n if (!connection) {\n return undefined;\n }\n\n if (connection.revision !== revision) {\n this.#lc.debug?.('Skipping validateConnection for stale revision', {\n clientID: selector.clientID,\n attemptedRevision: revision,\n currentRevision: connection.revision,\n });\n return undefined;\n }\n\n let validatedUserState: UserState | undefined;\n\n // If the API server has validated the user's identity, we ensure that\n // the connection's claimed userID matches it.\n if (validation.kind === 'server-validated') {\n validatedUserState = {id: validation.validatedUserID};\n\n // Check that the ws connection userID provided by the client\n // matches the validated userID from the API server.\n if (connection.user.id !== validatedUserState.id) {\n throw new ProtocolErrorWithLevel(\n {\n kind: ErrorKind.Unauthorized,\n message:\n 'Connection userID does not match validated server userID.',\n origin: ErrorOrigin.ZeroCache,\n },\n 'warn',\n );\n }\n }\n\n // The incoming user state is either the validated user state from the server\n // or the WS client's claimed user state if no server validation occurred.\n const incomingUserState = validatedUserState ?? connection.user;\n\n // Once a client group is validated, every later validated connection must\n // agree with that pinned identity.\n if (\n this.#group.pinnedUser !== undefined &&\n this.#group.pinnedUser.id !== incomingUserState.id\n ) {\n throw new ProtocolErrorWithLevel(\n {\n kind: ErrorKind.Unauthorized,\n message:\n 'Client groups are pinned to a single userID. Connection userID does not match existing client group userID.',\n origin: ErrorOrigin.ZeroCache,\n },\n 'warn',\n );\n }\n\n if (this.#group.pinnedUser === undefined) {\n this.#setGroup({\n ...this.#group,\n pinnedUser: incomingUserState,\n });\n }\n\n const validatedConnection = this.#storeConnection({\n ...connection,\n state: 'validated',\n revalidateAt: this.#nextRevalidateAt(),\n });\n this.#refreshBackgroundConnectionContext(validatedConnection);\n this.#updateBackgroundRetransformDeadline(false);\n\n return {\n connection: validatedConnection,\n group: this.getGroupState(),\n };\n }\n\n /** Removes one connection due to failed auth and updates all derived background/deadline state. */\n failConnection(\n selector: ConnectionSelector,\n revision: number,\n ): ConnectionContext | undefined {\n return this.#removeConnection(selector, revision);\n }\n\n /** Removes one disconnected connection and updates all derived background/deadline state. */\n closeConnection(selector: ConnectionSelector): ConnectionContext | undefined {\n return this.#removeConnection(selector);\n }\n\n /**\n * Records a successful background retransform. This starts a fresh interval\n * from the manager clock when shared retransform is schedulable, or\n * clears the deadline if it is not.\n */\n markBackgroundRetransformSuccess(\n selector: ConnectionSelector,\n revision: number,\n ): void {\n const backgroundConnection = this.#getBackgroundConnectionContext();\n if (!backgroundConnection) {\n return;\n }\n if (\n backgroundConnection.clientID !== selector.clientID ||\n backgroundConnection.wsID !== selector.wsID ||\n backgroundConnection.revision !== revision\n ) {\n return;\n }\n this.#updateBackgroundRetransformDeadline(true);\n }\n\n setSharedRetransformReady(ready: boolean): void {\n if (this.#sharedRetransformReady === ready) {\n return;\n }\n this.#sharedRetransformReady = ready;\n this.#updateBackgroundRetransformDeadline(true);\n }\n\n deferMaintenance(kind: 'revalidate' | 'retransform'): void {\n const intervalMs =\n kind === 'revalidate'\n ? this.#revalidateIntervalMs\n : this.#retransformIntervalMs;\n if (intervalMs === undefined) {\n return;\n }\n this.#setGroup({\n ...this.#group,\n maintenanceNotBeforeAt: Math.max(\n this.#group.maintenanceNotBeforeAt ?? 0,\n this.#now() + intervalMs,\n ),\n });\n }\n\n /** Returns the current live record for a client slot, if any. */\n getConnectionContext(\n selector: ConnectionSelector,\n ): Readonly<ConnectionContext> | undefined {\n return this.#getConnectionContext(selector);\n }\n\n /** Returns the live record for one websocket or throws if it is unavailable. */\n mustGetConnectionContext(\n selector: ConnectionSelector,\n ): Readonly<ConnectionContext> {\n return this.#mustGetConnectionContext(selector);\n }\n\n /** Returns the current background connection, if one exists. */\n getBackgroundConnectionContext(): Readonly<ConnectionContext> | undefined {\n return this.#getBackgroundConnectionContext();\n }\n\n mustGetBackgroundConnectionContext(): Readonly<ConnectionContext> {\n const backgroundConnection = this.#getBackgroundConnectionContext();\n if (!backgroundConnection) {\n throw new ProtocolErrorWithLevel(\n {\n kind: ErrorKind.InvalidConnectionRequest,\n message:\n 'No validated connection is available for shared query work.',\n origin: ErrorOrigin.ZeroCache,\n },\n 'warn',\n );\n }\n return backgroundConnection;\n }\n\n /** Returns the shared group auth state. */\n getGroupState(): Readonly<GroupAuthState> {\n return this.#group;\n }\n\n /**\n * Reports which maintenance work is currently due.\n *\n * The result is a pure snapshot: callers decide which actions to run and\n * when to wake up next. `earliestDeadlineAt` is the earliest outstanding\n * maintenance deadline, including overdue work, unless a transient failure\n * has deferred all scheduled maintenance until `maintenanceNotBeforeAt`.\n */\n planMaintenance(): {\n dueRevalidations: Readonly<ConnectionContext>[];\n dueRetransform: boolean;\n earliestDeadlineAt: number | undefined;\n } {\n const dueRevalidations: Readonly<ConnectionContext>[] = [];\n const now = this.#now();\n let earliestDeadlineAt = this.#group.retransformAt;\n\n for (const connection of this.#connections.values()) {\n if (\n connection.state !== 'validated' ||\n connection.revalidateAt === undefined\n ) {\n continue;\n }\n if (connection.revalidateAt <= now) {\n dueRevalidations.push(connection);\n }\n earliestDeadlineAt = minDefined(\n earliestDeadlineAt,\n connection.revalidateAt,\n );\n }\n\n const dueRetransform =\n this.#group.retransformAt !== undefined &&\n this.#group.retransformAt <= now;\n const maintenanceNotBeforeAt = this.#group.maintenanceNotBeforeAt;\n\n if (\n maintenanceNotBeforeAt !== undefined &&\n maintenanceNotBeforeAt > now &&\n earliestDeadlineAt !== undefined\n ) {\n return {\n dueRevalidations: [],\n dueRetransform: false,\n earliestDeadlineAt: Math.max(\n earliestDeadlineAt,\n maintenanceNotBeforeAt,\n ),\n };\n }\n\n return {\n dueRevalidations: dueRevalidations.sort(compareByInsertionOrder),\n dueRetransform,\n earliestDeadlineAt,\n };\n }\n\n #removeConnection(\n selector: ConnectionSelector,\n revision?: number,\n ): Readonly<ConnectionContext> | undefined {\n const connection = this.#getConnectionContext(selector);\n\n if (!connection) {\n return undefined;\n }\n\n // If the revision has changed, we should not remove the connection\n if (revision !== undefined && connection.revision !== revision) {\n this.#lc.debug?.('Ignoring failConnection for stale revision', {\n clientID: selector.clientID,\n wsID: selector.wsID,\n attemptedRevision: revision,\n currentRevision: connection.revision,\n });\n return undefined;\n }\n\n this.#connections.delete(connection.clientID);\n this.#refreshBackgroundConnectionContext();\n this.#updateBackgroundRetransformDeadline(false);\n\n return connection;\n }\n\n #demoteConnection(connection: ConnectionContext): ConnectionContext {\n const demotedConnection = this.#storeConnection({\n ...connection,\n state: 'provisional',\n revalidateAt: undefined,\n });\n this.#refreshBackgroundConnectionContext();\n this.#updateBackgroundRetransformDeadline(false);\n return demotedConnection;\n }\n\n /**\n * Keeps the background connection sticky while it remains validated.\n *\n * When a newly validated `preferred` connection is provided, it is promoted\n * only if there is no current validated background connection. Otherwise the\n * existing background connection stays in place until it disappears or is\n * demoted, at which point the newest validated connection is selected.\n */\n #refreshBackgroundConnectionContext(preferred?: ConnectionContext): void {\n if (preferred?.state === 'validated') {\n const currentBackgroundConnection =\n this.#getBackgroundConnectionContext();\n if (\n currentBackgroundConnection?.clientID === preferred.clientID &&\n currentBackgroundConnection.wsID === preferred.wsID\n ) {\n return;\n }\n if (currentBackgroundConnection !== undefined) {\n return;\n }\n this.#setBackgroundConnection({\n clientID: preferred.clientID,\n wsID: preferred.wsID,\n });\n this.#lc.debug?.('Selected background connection for shared auth work', {\n clientID: preferred.clientID,\n wsID: preferred.wsID,\n revision: preferred.revision,\n reason: 'preferred-validated',\n });\n return;\n }\n\n const currentBackgroundConnection = this.#getBackgroundConnectionContext();\n if (currentBackgroundConnection?.state === 'validated') {\n return;\n }\n\n const nextBackgroundConnection = [...this.#connections.values()]\n .filter(connection => connection.state === 'validated')\n .sort(comparePreferredValidatedConnection)\n .at(0);\n this.#setBackgroundConnection(\n nextBackgroundConnection\n ? {\n clientID: nextBackgroundConnection.clientID,\n wsID: nextBackgroundConnection.wsID,\n }\n : undefined,\n );\n if (nextBackgroundConnection) {\n this.#lc.debug?.('Selected background connection for shared auth work', {\n clientID: nextBackgroundConnection.clientID,\n wsID: nextBackgroundConnection.wsID,\n revision: nextBackgroundConnection.revision,\n reason: 'fallback-validated',\n });\n }\n }\n\n #getBackgroundConnectionContext(): ConnectionContext | undefined {\n const backgroundConnection = this.#group.backgroundConnection;\n if (!backgroundConnection) {\n return undefined;\n }\n return this.#getConnectionContext(backgroundConnection);\n }\n\n #getConnectionContext(\n selector: ConnectionSelector,\n ): ConnectionContext | undefined {\n const connection = this.#connections.get(selector.clientID);\n if (!connection) {\n return undefined;\n }\n if (connection.wsID !== selector.wsID) {\n return undefined;\n }\n return connection;\n }\n\n #mustGetConnectionContext(selector: ConnectionSelector): ConnectionContext {\n const connection = this.#getConnectionContext(selector);\n\n if (!connection) {\n throw new ProtocolErrorWithLevel(\n {\n kind: ErrorKind.InvalidConnectionRequest,\n message:\n 'Connection auth state was not available for this websocket.',\n origin: ErrorOrigin.ZeroCache,\n },\n 'warn',\n );\n }\n\n return connection;\n }\n\n #storeConnection(connection: ConnectionContext): ConnectionContext {\n this.#connections.set(connection.clientID, connection);\n return connection;\n }\n\n #setGroup(group: GroupAuthState): GroupAuthState {\n this.#group = group;\n return group;\n }\n\n #setBackgroundConnection(\n backgroundConnection: ConnectionSelector | undefined,\n ) {\n if (\n sameConnectionSelector(\n this.#group.backgroundConnection,\n backgroundConnection,\n )\n ) {\n return;\n }\n this.#setGroup({\n ...this.#group,\n backgroundConnection: backgroundConnection\n ? {...backgroundConnection}\n : undefined,\n });\n }\n\n /**\n * Keeps the group background retransform deadline coherent with current\n * schedulability.\n *\n * When `reset` is false, this seeds a deadline only when shared retransform\n * is now possible and no deadline exists yet, preserving any existing\n * cadence. When `reset` is true, it starts a fresh interval from `#now()` if\n * retransform is schedulable for the current ready ViewSyncer instance, or\n * clears the deadline if it is not.\n */\n #updateBackgroundRetransformDeadline(reset: boolean) {\n const backgroundConnection = this.#getBackgroundConnectionContext();\n if (\n !backgroundConnection ||\n this.#retransformIntervalMs === undefined ||\n !this.#sharedRetransformReady\n ) {\n if (this.#group.retransformAt !== undefined) {\n this.#setGroup({\n ...this.#group,\n retransformAt: undefined,\n });\n }\n return;\n }\n\n if (reset || this.#group.retransformAt === undefined) {\n this.#setGroup({\n ...this.#group,\n retransformAt: this.#now() + this.#retransformIntervalMs,\n });\n }\n }\n\n #nextRevalidateAt() {\n return this.#revalidateIntervalMs === undefined\n ? undefined\n : this.#now() + this.#revalidateIntervalMs;\n }\n}\n\nfunction compareByInsertionOrder(\n a: Pick<ConnectionContext, 'insertionOrder' | 'wsID'>,\n b: Pick<ConnectionContext, 'insertionOrder' | 'wsID'>,\n) {\n return a.insertionOrder - b.insertionOrder || a.wsID.localeCompare(b.wsID);\n}\n\nfunction comparePreferredValidatedConnection(\n a: Pick<ConnectionContext, 'insertionOrder' | 'wsID'>,\n b: Pick<ConnectionContext, 'insertionOrder' | 'wsID'>,\n) {\n return b.insertionOrder - a.insertionOrder || b.wsID.localeCompare(a.wsID);\n}\n\nfunction minDefined(a: number | undefined, b: number | undefined) {\n if (a === undefined) {\n return b;\n }\n if (b === undefined) {\n return a;\n }\n return Math.min(a, b);\n}\n\nfunction sameConnectionSelector(\n a: ConnectionSelector | undefined,\n b: ConnectionSelector | undefined,\n) {\n return a?.clientID === b?.clientID && a?.wsID === b?.wsID;\n}\n\nfunction filterHeaders(\n headers: Readonly<Record<string, string>> | undefined,\n allowedHeaders: readonly string[] | undefined,\n) {\n if (!headers || !allowedHeaders || allowedHeaders.length === 0) {\n return undefined;\n }\n\n const allowed = new Set(allowedHeaders.map(header => header.toLowerCase()));\n let filtered: Record<string, string> | undefined;\n for (const [key, value] of Object.entries(headers)) {\n if (allowed.has(key.toLowerCase())) {\n filtered ??= {};\n filtered[key] = value;\n }\n }\n return filtered;\n}\n"],"mappings":";;;;;;;;;;;;;;AAmLA,IAAa,+BAAb,MAA8E;CAC5E;CAGA,+BAAwB,IAAI,IAA+B;CAC3D,SAAyB;EACvB,YAAY,KAAA;EACZ,sBAAsB,KAAA;EACtB,eAAe,KAAA;EACf,wBAAwB,KAAA;CAC1B;CAEA;CAEA;CACA;CACA;CACA;CACA;CACA,0BAA0B;CAC1B,sBAAsB;CAEtB,YACE,IACA,2BACA,4BACA,aACA,YACA,mBACA,KACA;EACA,KAAKA,MAAM;EACX,KAAKG,OAAO,OAAO,KAAK;EACxB,KAAKC,wBACH,8BAA8B,KAAA,IAC1B,KAAA,IACA,4BAA4B;EAClC,KAAKC,yBACH,+BAA+B,KAAA,IAC3B,KAAA,IACA,6BAA6B;EACnC,KAAKC,eAAe;EACpB,KAAKC,cAAc;EACnB,KAAKL,qBAAqB;CAC5B;;;;;;;CAQA,mBACE,UACA,eACA,MAC6B;EAC7B,KAAKM,kBAAkB,QAAQ;EAE/B,MAAM,cAAc,SAAqD;GACvE,MAAM,SAAS,SAAS,UAAU,KAAKF,eAAe,KAAKC;GAE3D,OAAO;IACL,KAAK,QAAQ,MAAM;IACnB,oBAAoB,QAAQ,KAAK,IAAI,iBAAiB;IACtD,eAAe;KACb,eAAe,KAAA;KACf,gBAAgB,cACd,cAAc,gBACd,QAAQ,qBACV;KACA,QAAQ,cAAc;KACtB,QAAQ,QAAQ;KAChB,QAAQ,QAAQ,iBAAiB,cAAc,aAAa,KAAA;IAC9D;GACF;EACF;EAEA,MAAM,aAAgC;GACpC,OAAO;GAEP,UAAU,cAAc;GACxB,MAAM,cAAc;GACpB,UAAU;GACV,MAAM,EAAC,IAAI,cAAc,UAAU,KAAI;GACvC;GAEA,WAAW,cAAc;GACzB,YAAY,cAAc;GAC1B,iBAAiB,cAAc;GAC/B,UAAU,cAAc;GAExB,cAAc,KAAA;GAEd,cAAc,WAAW,OAAO;GAChC,eAAe,WAAW,QAAQ;GAElC,gBAAgB,EAAE,KAAKE;EACzB;EACA,KAAKC,iBAAiB,UAAU;EAChC,KAAKC,oCAAoC;EACzC,KAAKC,qCAAqC,KAAK;EAC/C,OAAO;CACT;;;;;;;CAQA,eACE,UACA,MAC6B;EAC7B,MAAM,aAAa,KAAKC,0BAA0B,QAAQ;EAE1D,IAAI,eAAe,WAAW;EAC9B,IAAI,gBAAgB,WAAW;EAE/B,IAAI,KAAK,cACP,eAAe;GACb,GAAG;GACH,KAAK,KAAK;EACZ;EAEF,IAAI,KAAK,kBACP,eAAe;GACb,GAAG;GACH,eAAe;IACb,GAAG,aAAa;IAChB,eAAe,cACb,KAAK,kBACL,KAAKP,cAAc,oBACrB;GACF;EACF;EAEF,IAAI,KAAK,aACP,gBAAgB;GACd,GAAG;GACH,KAAK,KAAK;EACZ;EAEF,IAAI,KAAK,iBACP,gBAAgB;GACd,GAAG;GACH,eAAe;IACb,GAAG,cAAc;IACjB,eAAe,cACb,KAAK,iBACL,KAAKC,aAAa,oBACpB;GACF;EACF;EAGF,OAAO,KAAKO,kBAAkB;GAC5B,GAAG;GACH,UAAU,WAAW,WAAW;GAChC;GACA;EACF,CAAC;CACH;;;;;CAMA,MAAM,WACJ,UACA,MACsC;EACtC,MAAM,aAAa,KAAKD,0BAA0B,QAAQ;EAE1D,MAAM,WAAW,MAAM,YACrB,KAAKb,KACL,WAAW,MACX,WAAW,KAAK,IAChB,KAAK,MACL,KAAKE,kBACP;EAGA,IAAI,CADiB,WAAW,WAAW,MAAM,QAAQ,GAEvD,OAAO,KAAKY,kBAAkB;GAC5B,GAAG;GACH,MAAM;GACN,UAAU,WAAW,WAAW;EAClC,CAAC;EAGH,IAAI,aAAa,WAAW,MAC1B,OAAO;EAGT,OAAO,KAAKJ,iBAAiB;GAC3B,GAAG;GACH,MAAM;EACR,CAAC;CACH;;;;;;;;;;CAWA,mBACE,UACA,UACA,YAMY;EACZ,MAAM,aAAa,KAAKK,sBAAsB,QAAQ;EACtD,IAAI,CAAC,YACH;EAGF,IAAI,WAAW,aAAa,UAAU;GACpC,KAAKf,IAAI,QAAQ,kDAAkD;IACjE,UAAU,SAAS;IACnB,mBAAmB;IACnB,iBAAiB,WAAW;GAC9B,CAAC;GACD;EACF;EAEA,IAAI;EAIJ,IAAI,WAAW,SAAS,oBAAoB;GAC1C,qBAAqB,EAAC,IAAI,WAAW,gBAAe;GAIpD,IAAI,WAAW,KAAK,OAAO,mBAAmB,IAC5C,MAAM,IAAI,uBACR;IACE,MAAM;IACN,SACE;IACF,QAAQ;GACV,GACA,MACF;EAEJ;EAIA,MAAM,oBAAoB,sBAAsB,WAAW;EAI3D,IACE,KAAKgB,OAAO,eAAe,KAAA,KAC3B,KAAKA,OAAO,WAAW,OAAO,kBAAkB,IAEhD,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SACE;GACF,QAAQ;EACV,GACA,MACF;EAGF,IAAI,KAAKA,OAAO,eAAe,KAAA,GAC7B,KAAKC,UAAU;GACb,GAAG,KAAKD;GACR,YAAY;EACd,CAAC;EAGH,MAAM,sBAAsB,KAAKN,iBAAiB;GAChD,GAAG;GACH,OAAO;GACP,cAAc,KAAKQ,kBAAkB;EACvC,CAAC;EACD,KAAKP,oCAAoC,mBAAmB;EAC5D,KAAKC,qCAAqC,KAAK;EAE/C,OAAO;GACL,YAAY;GACZ,OAAO,KAAK,cAAc;EAC5B;CACF;;CAGA,eACE,UACA,UAC+B;EAC/B,OAAO,KAAKJ,kBAAkB,UAAU,QAAQ;CAClD;;CAGA,gBAAgB,UAA6D;EAC3E,OAAO,KAAKA,kBAAkB,QAAQ;CACxC;;;;;;CAOA,iCACE,UACA,UACM;EACN,MAAM,uBAAuB,KAAKW,gCAAgC;EAClE,IAAI,CAAC,sBACH;EAEF,IACE,qBAAqB,aAAa,SAAS,YAC3C,qBAAqB,SAAS,SAAS,QACvC,qBAAqB,aAAa,UAElC;EAEF,KAAKP,qCAAqC,IAAI;CAChD;CAEA,0BAA0B,OAAsB;EAC9C,IAAI,KAAKQ,4BAA4B,OACnC;EAEF,KAAKA,0BAA0B;EAC/B,KAAKR,qCAAqC,IAAI;CAChD;CAEA,iBAAiB,MAA0C;EACzD,MAAM,aACJ,SAAS,eACL,KAAKR,wBACL,KAAKC;EACX,IAAI,eAAe,KAAA,GACjB;EAEF,KAAKY,UAAU;GACb,GAAG,KAAKD;GACR,wBAAwB,KAAK,IAC3B,KAAKA,OAAO,0BAA0B,GACtC,KAAKb,KAAK,IAAI,UAChB;EACF,CAAC;CACH;;CAGA,qBACE,UACyC;EACzC,OAAO,KAAKY,sBAAsB,QAAQ;CAC5C;;CAGA,yBACE,UAC6B;EAC7B,OAAO,KAAKF,0BAA0B,QAAQ;CAChD;;CAGA,iCAA0E;EACxE,OAAO,KAAKM,gCAAgC;CAC9C;CAEA,qCAAkE;EAChE,MAAM,uBAAuB,KAAKA,gCAAgC;EAClE,IAAI,CAAC,sBACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SACE;GACF,QAAQ;EACV,GACA,MACF;EAEF,OAAO;CACT;;CAGA,gBAA0C;EACxC,OAAO,KAAKH;CACd;;;;;;;;;CAUA,kBAIE;EACA,MAAM,mBAAkD,CAAC;EACzD,MAAM,MAAM,KAAKb,KAAK;EACtB,IAAI,qBAAqB,KAAKa,OAAO;EAErC,KAAK,MAAM,cAAc,KAAKf,aAAa,OAAO,GAAG;GACnD,IACE,WAAW,UAAU,eACrB,WAAW,iBAAiB,KAAA,GAE5B;GAEF,IAAI,WAAW,gBAAgB,KAC7B,iBAAiB,KAAK,UAAU;GAElC,qBAAqB,WACnB,oBACA,WAAW,YACb;EACF;EAEA,MAAM,iBACJ,KAAKe,OAAO,kBAAkB,KAAA,KAC9B,KAAKA,OAAO,iBAAiB;EAC/B,MAAM,yBAAyB,KAAKA,OAAO;EAE3C,IACE,2BAA2B,KAAA,KAC3B,yBAAyB,OACzB,uBAAuB,KAAA,GAEvB,OAAO;GACL,kBAAkB,CAAC;GACnB,gBAAgB;GAChB,oBAAoB,KAAK,IACvB,oBACA,sBACF;EACF;EAGF,OAAO;GACL,kBAAkB,iBAAiB,KAAK,uBAAuB;GAC/D;GACA;EACF;CACF;CAEA,kBACE,UACA,UACyC;EACzC,MAAM,aAAa,KAAKD,sBAAsB,QAAQ;EAEtD,IAAI,CAAC,YACH;EAIF,IAAI,aAAa,KAAA,KAAa,WAAW,aAAa,UAAU;GAC9D,KAAKf,IAAI,QAAQ,8CAA8C;IAC7D,UAAU,SAAS;IACnB,MAAM,SAAS;IACf,mBAAmB;IACnB,iBAAiB,WAAW;GAC9B,CAAC;GACD;EACF;EAEA,KAAKC,aAAa,OAAO,WAAW,QAAQ;EAC5C,KAAKU,oCAAoC;EACzC,KAAKC,qCAAqC,KAAK;EAE/C,OAAO;CACT;CAEA,kBAAkB,YAAkD;EAClE,MAAM,oBAAoB,KAAKF,iBAAiB;GAC9C,GAAG;GACH,OAAO;GACP,cAAc,KAAA;EAChB,CAAC;EACD,KAAKC,oCAAoC;EACzC,KAAKC,qCAAqC,KAAK;EAC/C,OAAO;CACT;;;;;;;;;CAUA,oCAAoC,WAAqC;EACvE,IAAI,WAAW,UAAU,aAAa;GACpC,MAAM,8BACJ,KAAKO,gCAAgC;GACvC,IACE,6BAA6B,aAAa,UAAU,YACpD,4BAA4B,SAAS,UAAU,MAE/C;GAEF,IAAI,gCAAgC,KAAA,GAClC;GAEF,KAAKE,yBAAyB;IAC5B,UAAU,UAAU;IACpB,MAAM,UAAU;GAClB,CAAC;GACD,KAAKrB,IAAI,QAAQ,uDAAuD;IACtE,UAAU,UAAU;IACpB,MAAM,UAAU;IAChB,UAAU,UAAU;IACpB,QAAQ;GACV,CAAC;GACD;EACF;EAGA,IADoC,KAAKmB,gCACrC,GAA6B,UAAU,aACzC;EAGF,MAAM,2BAA2B,CAAC,GAAG,KAAKlB,aAAa,OAAO,CAAC,EAC5D,QAAO,eAAc,WAAW,UAAU,WAAW,EACrD,KAAK,mCAAmC,EACxC,GAAG,CAAC;EACP,KAAKoB,yBACH,2BACI;GACE,UAAU,yBAAyB;GACnC,MAAM,yBAAyB;EACjC,IACA,KAAA,CACN;EACA,IAAI,0BACF,KAAKrB,IAAI,QAAQ,uDAAuD;GACtE,UAAU,yBAAyB;GACnC,MAAM,yBAAyB;GAC/B,UAAU,yBAAyB;GACnC,QAAQ;EACV,CAAC;CAEL;CAEA,kCAAiE;EAC/D,MAAM,uBAAuB,KAAKgB,OAAO;EACzC,IAAI,CAAC,sBACH;EAEF,OAAO,KAAKD,sBAAsB,oBAAoB;CACxD;CAEA,sBACE,UAC+B;EAC/B,MAAM,aAAa,KAAKd,aAAa,IAAI,SAAS,QAAQ;EAC1D,IAAI,CAAC,YACH;EAEF,IAAI,WAAW,SAAS,SAAS,MAC/B;EAEF,OAAO;CACT;CAEA,0BAA0B,UAAiD;EACzE,MAAM,aAAa,KAAKc,sBAAsB,QAAQ;EAEtD,IAAI,CAAC,YACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SACE;GACF,QAAQ;EACV,GACA,MACF;EAGF,OAAO;CACT;CAEA,iBAAiB,YAAkD;EACjE,KAAKd,aAAa,IAAI,WAAW,UAAU,UAAU;EACrD,OAAO;CACT;CAEA,UAAU,OAAuC;EAC/C,KAAKe,SAAS;EACd,OAAO;CACT;CAEA,yBACE,sBACA;EACA,IACE,uBACE,KAAKA,OAAO,sBACZ,oBACF,GAEA;EAEF,KAAKC,UAAU;GACb,GAAG,KAAKD;GACR,sBAAsB,uBAClB,EAAC,GAAG,qBAAoB,IACxB,KAAA;EACN,CAAC;CACH;;;;;;;;;;;CAYA,qCAAqC,OAAgB;EAEnD,IACE,CAF2B,KAAKG,gCAE/B,KACD,KAAKd,2BAA2B,KAAA,KAChC,CAAC,KAAKe,yBACN;GACA,IAAI,KAAKJ,OAAO,kBAAkB,KAAA,GAChC,KAAKC,UAAU;IACb,GAAG,KAAKD;IACR,eAAe,KAAA;GACjB,CAAC;GAEH;EACF;EAEA,IAAI,SAAS,KAAKA,OAAO,kBAAkB,KAAA,GACzC,KAAKC,UAAU;GACb,GAAG,KAAKD;GACR,eAAe,KAAKb,KAAK,IAAI,KAAKE;EACpC,CAAC;CAEL;CAEA,oBAAoB;EAClB,OAAO,KAAKD,0BAA0B,KAAA,IAClC,KAAA,IACA,KAAKD,KAAK,IAAI,KAAKC;CACzB;AACF;AAEA,SAAS,wBACP,GACA,GACA;CACA,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,KAAK,cAAc,EAAE,IAAI;AAC3E;AAEA,SAAS,oCACP,GACA,GACA;CACA,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,KAAK,cAAc,EAAE,IAAI;AAC3E;AAEA,SAAS,WAAW,GAAuB,GAAuB;CAChE,IAAI,MAAM,KAAA,GACR,OAAO;CAET,IAAI,MAAM,KAAA,GACR,OAAO;CAET,OAAO,KAAK,IAAI,GAAG,CAAC;AACtB;AAEA,SAAS,uBACP,GACA,GACA;CACA,OAAO,GAAG,aAAa,GAAG,YAAY,GAAG,SAAS,GAAG;AACvD;AAEA,SAAS,cACP,SACA,gBACA;CACA,IAAI,CAAC,WAAW,CAAC,kBAAkB,eAAe,WAAW,GAC3D;CAGF,MAAM,UAAU,IAAI,IAAI,eAAe,KAAI,WAAU,OAAO,YAAY,CAAC,CAAC;CAC1E,IAAI;CACJ,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,GAC/C,IAAI,QAAQ,IAAI,IAAI,YAAY,CAAC,GAAG;EAClC,aAAa,CAAC;EACd,SAAS,OAAO;CAClB;CAEF,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"connection-context-manager.js","names":["#lc","#connections","#validateLegacyJWT","#now","#revalidateIntervalMs","#retransformIntervalMs","#queryConfig","#pushConfig","#removeConnection","#nextInsertionOrder","#storeConnection","#refreshBackgroundConnectionContext","#updateBackgroundRetransformDeadline","#mustGetConnectionContext","#demoteConnection","#getConnectionContext","#group","#setGroup","#nextRevalidateAt","#getBackgroundConnectionContext","#sharedRetransformReady","#setBackgroundConnection"],"sources":["../../../../../../zero-cache/src/services/view-syncer/connection-context-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {InitConnectionBody} from '../../../../zero-protocol/src/connect.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport type {ProtocolFlags} from '../../../../zero-protocol/src/protocol-flag.ts';\nimport type {UpdateAuthBody} from '../../../../zero-protocol/src/update-auth.ts';\nimport {\n authEquals,\n resolveAuth,\n type Auth,\n type ValidateLegacyJWT,\n} from '../../auth/auth.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {compileUrlPattern} from '../../custom/fetch.ts';\nimport {ProtocolErrorWithLevel} from '../../types/error-with-level.ts';\nimport type {ConnectParams} from '../../workers/connect-params.ts';\n\nexport type ConnectionState = 'provisional' | 'validated';\n\n/**\n * Normalized user identity shared by live connection state and group auth state.\n * `id: null` means logged out.\n */\nexport type UserState = {readonly id: string | null};\n\n/**\n * Delineates the two paths for validating a connection: either server can validate\n * the user's identity and return a definitive userID to trust, or we fall back to\n * trusting the one provided by the client in the incoming query params.\n */\nexport type ConnectionValidation =\n | {kind: 'client-fallback'}\n | {kind: 'server-validated'; validatedUserID: string | null};\n\n/**\n * Identifies one live websocket for a client slot.\n */\nexport type ConnectionSelector = {\n readonly clientID: string;\n readonly wsID: string;\n};\n\ntype FetchConfig = ZeroConfig['query'];\n\nexport type HeaderOptions = {\n readonly apiKey?: string | undefined;\n /** Allowlisted headers provided in the client options. */\n readonly customHeaders?: Readonly<Record<string, string>> | undefined;\n /** Allowlisted headers from the incoming HTTP request. */\n readonly requestHeaders?: Readonly<Record<string, string>> | undefined;\n readonly cookie?: string | undefined;\n readonly origin?: string | undefined;\n};\n\nexport type ConnectionFetchContext = {\n readonly url: string | undefined;\n readonly allowedUrlPatterns: readonly URLPattern[] | undefined;\n readonly headerOptions: HeaderOptions;\n};\n\n/**\n * A snapshot of one live connection tracked by the manager.\n *\n * `revalidateAt` is only populated while the connection is `validated`.\n */\nexport type ConnectionContext = {\n readonly state: ConnectionState;\n\n readonly clientID: string;\n readonly wsID: string;\n readonly user: UserState;\n\n readonly auth: Auth | undefined;\n\n readonly profileID: string | null;\n readonly baseCookie: string | null;\n readonly protocolVersion: number;\n readonly protocolFlags: ProtocolFlags;\n\n readonly revision: number;\n\n readonly revalidateAt: number | undefined;\n\n readonly insertionOrder: number;\n\n readonly queryContext: ConnectionFetchContext;\n readonly mutateContext: ConnectionFetchContext;\n};\n\n/**\n * Group-scoped auth state shared across the live connections.\n *\n * The background connection is the validated connection currently used for\n * shared background work. Retransform happens on a group level, and uses\n * the background connection's credential to refetch the latest queries.\n */\nexport type GroupAuthState = {\n readonly pinnedUser: UserState | undefined;\n\n readonly backgroundConnection: ConnectionSelector | undefined;\n readonly retransformAt: number | undefined;\n // Defer all maintenance in case a transient failure occurs.\n readonly maintenanceNotBeforeAt: number | undefined;\n};\n\nexport type ConnectionContextManager = {\n registerConnection(\n selector: ConnectionSelector,\n connectParams: ConnectParams,\n auth?: Auth,\n ): Readonly<ConnectionContext>;\n\n initConnection(\n selector: ConnectionSelector,\n body: InitConnectionBody,\n ): Readonly<ConnectionContext>;\n\n updateAuth(\n selector: ConnectionSelector,\n body: UpdateAuthBody,\n ): Promise<Readonly<ConnectionContext>>;\n\n validateConnection(\n selector: ConnectionSelector,\n revision: number,\n validation: ConnectionValidation,\n ):\n | Readonly<{\n connection: ConnectionContext;\n group: GroupAuthState;\n }>\n | undefined;\n\n failConnection(\n selector: ConnectionSelector,\n revision: number,\n ): Readonly<ConnectionContext> | undefined;\n closeConnection(\n selector: ConnectionSelector,\n ): Readonly<ConnectionContext> | undefined;\n\n markBackgroundRetransformSuccess(\n selector: ConnectionSelector,\n revision: number,\n ): void;\n\n setSharedRetransformReady(ready: boolean): void;\n\n deferMaintenance(kind: 'revalidate' | 'retransform'): void;\n\n getConnectionContext(\n selector: ConnectionSelector,\n ): Readonly<ConnectionContext> | undefined;\n mustGetConnectionContext(\n selector: ConnectionSelector,\n ): Readonly<ConnectionContext>;\n\n getBackgroundConnectionContext(): Readonly<ConnectionContext> | undefined;\n mustGetBackgroundConnectionContext(): Readonly<ConnectionContext>;\n\n getGroupState(): Readonly<GroupAuthState>;\n\n planMaintenance(): {\n dueRevalidations: Readonly<ConnectionContext>[];\n dueRetransform: boolean;\n earliestDeadlineAt: number | undefined;\n };\n};\n\n/**\n * State machine for the auth state of a single `ViewSyncerService`.\n *\n * Connections are registered as `provisional`, optionally backfilled with\n * `initConnection` metadata, and then promoted to `validated` once their\n * effective `userID` is confirmed as valid. The manager also tracks which\n * validated connection currently serves as the group's background connection.\n */\nexport class ConnectionContextManagerImpl implements ConnectionContextManager {\n readonly #lc: LogContext;\n\n // The live connection records, keyed by clientID\n readonly #connections = new Map<string, ConnectionContext>();\n #group: GroupAuthState = {\n pinnedUser: undefined,\n backgroundConnection: undefined,\n retransformAt: undefined,\n maintenanceNotBeforeAt: undefined,\n };\n\n readonly #validateLegacyJWT: ValidateLegacyJWT | undefined;\n\n readonly #now: () => number;\n readonly #revalidateIntervalMs: number | undefined;\n readonly #retransformIntervalMs: number | undefined;\n readonly #queryConfig: FetchConfig | undefined;\n readonly #pushConfig: FetchConfig | undefined;\n #sharedRetransformReady = false;\n #nextInsertionOrder = 0;\n\n constructor(\n lc: LogContext,\n revalidateIntervalSeconds?: number,\n retransformIntervalSeconds?: number,\n queryConfig?: FetchConfig,\n pushConfig?: FetchConfig,\n validateLegacyJWT?: ValidateLegacyJWT,\n now?: () => number,\n ) {\n this.#lc = lc;\n this.#now = now ?? Date.now;\n this.#revalidateIntervalMs =\n revalidateIntervalSeconds === undefined\n ? undefined\n : revalidateIntervalSeconds * 1000;\n this.#retransformIntervalMs =\n retransformIntervalSeconds === undefined\n ? undefined\n : retransformIntervalSeconds * 1000;\n this.#queryConfig = queryConfig;\n this.#pushConfig = pushConfig;\n this.#validateLegacyJWT = validateLegacyJWT;\n }\n\n /**\n * Creates or replaces the live record for a websocket connection.\n *\n * Re-registering the same `clientID` drops the old socket record and starts\n * the replacement back in `provisional` state.\n */\n registerConnection(\n selector: ConnectionSelector,\n connectParams: ConnectParams,\n auth?: Auth,\n ): Readonly<ConnectionContext> {\n this.#removeConnection(selector);\n\n const getContext = (type: 'query' | 'mutate'): ConnectionFetchContext => {\n const config = type === 'query' ? this.#queryConfig : this.#pushConfig;\n\n return {\n url: config?.url?.[0],\n allowedUrlPatterns: config?.url?.map(compileUrlPattern),\n headerOptions: {\n customHeaders: undefined,\n requestHeaders: filterHeaders(\n connectParams.requestHeaders,\n config?.allowedRequestHeaders,\n ),\n origin: connectParams.origin,\n apiKey: config?.apiKey,\n cookie: config?.forwardCookies ? connectParams.httpCookie : undefined,\n },\n };\n };\n\n const connection: ConnectionContext = {\n state: 'provisional',\n\n clientID: connectParams.clientID,\n wsID: connectParams.wsID,\n revision: 0,\n user: {id: connectParams.userID ?? null},\n auth,\n\n profileID: connectParams.profileID,\n baseCookie: connectParams.baseCookie,\n protocolVersion: connectParams.protocolVersion,\n protocolFlags: connectParams.protocolFlags,\n\n revalidateAt: undefined,\n\n queryContext: getContext('query'),\n mutateContext: getContext('mutate'),\n\n insertionOrder: ++this.#nextInsertionOrder,\n };\n this.#storeConnection(connection);\n this.#refreshBackgroundConnectionContext();\n this.#updateBackgroundRetransformDeadline(false);\n return connection;\n }\n\n /**\n * Backfills `initConnection` data for sockets that were registered before the\n * client could send its full init payload.\n *\n * This updates metadata only; it does not validate the connection.\n */\n initConnection(\n selector: ConnectionSelector,\n body: InitConnectionBody,\n ): Readonly<ConnectionContext> {\n const connection = this.#mustGetConnectionContext(selector);\n\n let queryContext = connection.queryContext;\n let mutateContext = connection.mutateContext;\n\n if (body.userQueryURL) {\n queryContext = {\n ...queryContext,\n url: body.userQueryURL,\n };\n }\n if (body.userQueryHeaders) {\n queryContext = {\n ...queryContext,\n headerOptions: {\n ...queryContext.headerOptions,\n customHeaders: filterHeaders(\n body.userQueryHeaders,\n this.#queryConfig?.allowedClientHeaders,\n ),\n },\n };\n }\n if (body.userPushURL) {\n mutateContext = {\n ...mutateContext,\n url: body.userPushURL,\n };\n }\n if (body.userPushHeaders) {\n mutateContext = {\n ...mutateContext,\n headerOptions: {\n ...mutateContext.headerOptions,\n customHeaders: filterHeaders(\n body.userPushHeaders,\n this.#pushConfig?.allowedClientHeaders,\n ),\n },\n };\n }\n\n return this.#demoteConnection({\n ...connection,\n revision: connection.revision + 1,\n queryContext,\n mutateContext,\n });\n }\n\n /**\n * A material auth change demotes the connection back to provisional until it\n * is validated again.\n */\n async updateAuth(\n selector: ConnectionSelector,\n body: UpdateAuthBody,\n ): Promise<Readonly<ConnectionContext>> {\n const connection = this.#mustGetConnectionContext(selector);\n\n const nextAuth = await resolveAuth(\n this.#lc,\n connection.auth,\n connection.user.id,\n body.auth,\n this.#validateLegacyJWT,\n );\n\n const authChanged = !authEquals(connection.auth, nextAuth);\n if (authChanged) {\n return this.#demoteConnection({\n ...connection,\n auth: nextAuth,\n revision: connection.revision + 1,\n });\n }\n\n if (nextAuth === connection.auth) {\n return connection;\n }\n\n return this.#storeConnection({\n ...connection,\n auth: nextAuth,\n });\n }\n\n /**\n * Validates one connection against the group's pinned `userID`.\n *\n * The first successful validation binds the group `userID`. Later\n * validations must match it. Validation also refreshes the connection's\n * revalidation deadline and may pick the connection as the group\n * background connection if none is currently available. If the websocket is\n * gone by the time async validation finishes, this becomes a no-op.\n */\n validateConnection(\n selector: ConnectionSelector,\n revision: number,\n validation: ConnectionValidation,\n ):\n | Readonly<{\n connection: ConnectionContext;\n group: GroupAuthState;\n }>\n | undefined {\n const connection = this.#getConnectionContext(selector);\n if (!connection) {\n return undefined;\n }\n\n if (connection.revision !== revision) {\n this.#lc.debug?.('Skipping validateConnection for stale revision', {\n clientID: selector.clientID,\n attemptedRevision: revision,\n currentRevision: connection.revision,\n });\n return undefined;\n }\n\n let validatedUserState: UserState | undefined;\n\n // If the API server has validated the user's identity, we ensure that\n // the connection's claimed userID matches it.\n if (validation.kind === 'server-validated') {\n validatedUserState = {id: validation.validatedUserID};\n\n // Check that the ws connection userID provided by the client\n // matches the validated userID from the API server.\n if (connection.user.id !== validatedUserState.id) {\n throw new ProtocolErrorWithLevel(\n {\n kind: ErrorKind.Unauthorized,\n message:\n 'Connection userID does not match validated server userID.',\n origin: ErrorOrigin.ZeroCache,\n },\n 'warn',\n );\n }\n }\n\n // The incoming user state is either the validated user state from the server\n // or the WS client's claimed user state if no server validation occurred.\n const incomingUserState = validatedUserState ?? connection.user;\n\n // Once a client group is validated, every later validated connection must\n // agree with that pinned identity.\n if (\n this.#group.pinnedUser !== undefined &&\n this.#group.pinnedUser.id !== incomingUserState.id\n ) {\n throw new ProtocolErrorWithLevel(\n {\n kind: ErrorKind.Unauthorized,\n message:\n 'Client groups are pinned to a single userID. Connection userID does not match existing client group userID.',\n origin: ErrorOrigin.ZeroCache,\n },\n 'warn',\n );\n }\n\n if (this.#group.pinnedUser === undefined) {\n this.#setGroup({\n ...this.#group,\n pinnedUser: incomingUserState,\n });\n }\n\n const validatedConnection = this.#storeConnection({\n ...connection,\n state: 'validated',\n revalidateAt: this.#nextRevalidateAt(),\n });\n this.#refreshBackgroundConnectionContext(validatedConnection);\n this.#updateBackgroundRetransformDeadline(false);\n\n return {\n connection: validatedConnection,\n group: this.getGroupState(),\n };\n }\n\n /** Removes one connection due to failed auth and updates all derived background/deadline state. */\n failConnection(\n selector: ConnectionSelector,\n revision: number,\n ): ConnectionContext | undefined {\n return this.#removeConnection(selector, revision);\n }\n\n /** Removes one disconnected connection and updates all derived background/deadline state. */\n closeConnection(selector: ConnectionSelector): ConnectionContext | undefined {\n return this.#removeConnection(selector);\n }\n\n /**\n * Records a successful background retransform. This starts a fresh interval\n * from the manager clock when shared retransform is schedulable, or\n * clears the deadline if it is not.\n */\n markBackgroundRetransformSuccess(\n selector: ConnectionSelector,\n revision: number,\n ): void {\n const backgroundConnection = this.#getBackgroundConnectionContext();\n if (!backgroundConnection) {\n return;\n }\n if (\n backgroundConnection.clientID !== selector.clientID ||\n backgroundConnection.wsID !== selector.wsID ||\n backgroundConnection.revision !== revision\n ) {\n return;\n }\n this.#updateBackgroundRetransformDeadline(true);\n }\n\n setSharedRetransformReady(ready: boolean): void {\n if (this.#sharedRetransformReady === ready) {\n return;\n }\n this.#sharedRetransformReady = ready;\n this.#updateBackgroundRetransformDeadline(true);\n }\n\n deferMaintenance(kind: 'revalidate' | 'retransform'): void {\n const intervalMs =\n kind === 'revalidate'\n ? this.#revalidateIntervalMs\n : this.#retransformIntervalMs;\n if (intervalMs === undefined) {\n return;\n }\n this.#setGroup({\n ...this.#group,\n maintenanceNotBeforeAt: Math.max(\n this.#group.maintenanceNotBeforeAt ?? 0,\n this.#now() + intervalMs,\n ),\n });\n }\n\n /** Returns the current live record for a client slot, if any. */\n getConnectionContext(\n selector: ConnectionSelector,\n ): Readonly<ConnectionContext> | undefined {\n return this.#getConnectionContext(selector);\n }\n\n /** Returns the live record for one websocket or throws if it is unavailable. */\n mustGetConnectionContext(\n selector: ConnectionSelector,\n ): Readonly<ConnectionContext> {\n return this.#mustGetConnectionContext(selector);\n }\n\n /** Returns the current background connection, if one exists. */\n getBackgroundConnectionContext(): Readonly<ConnectionContext> | undefined {\n return this.#getBackgroundConnectionContext();\n }\n\n mustGetBackgroundConnectionContext(): Readonly<ConnectionContext> {\n const backgroundConnection = this.#getBackgroundConnectionContext();\n if (!backgroundConnection) {\n throw new ProtocolErrorWithLevel(\n {\n kind: ErrorKind.InvalidConnectionRequest,\n message:\n 'No validated connection is available for shared query work.',\n origin: ErrorOrigin.ZeroCache,\n },\n 'warn',\n );\n }\n return backgroundConnection;\n }\n\n /** Returns the shared group auth state. */\n getGroupState(): Readonly<GroupAuthState> {\n return this.#group;\n }\n\n /**\n * Reports which maintenance work is currently due.\n *\n * The result is a pure snapshot: callers decide which actions to run and\n * when to wake up next. `earliestDeadlineAt` is the earliest outstanding\n * maintenance deadline, including overdue work, unless a transient failure\n * has deferred all scheduled maintenance until `maintenanceNotBeforeAt`.\n */\n planMaintenance(): {\n dueRevalidations: Readonly<ConnectionContext>[];\n dueRetransform: boolean;\n earliestDeadlineAt: number | undefined;\n } {\n const dueRevalidations: Readonly<ConnectionContext>[] = [];\n const now = this.#now();\n let earliestDeadlineAt = this.#group.retransformAt;\n\n for (const connection of this.#connections.values()) {\n if (\n connection.state !== 'validated' ||\n connection.revalidateAt === undefined\n ) {\n continue;\n }\n if (connection.revalidateAt <= now) {\n dueRevalidations.push(connection);\n }\n earliestDeadlineAt = minDefined(\n earliestDeadlineAt,\n connection.revalidateAt,\n );\n }\n\n const dueRetransform =\n this.#group.retransformAt !== undefined &&\n this.#group.retransformAt <= now;\n const maintenanceNotBeforeAt = this.#group.maintenanceNotBeforeAt;\n\n if (\n maintenanceNotBeforeAt !== undefined &&\n maintenanceNotBeforeAt > now &&\n earliestDeadlineAt !== undefined\n ) {\n return {\n dueRevalidations: [],\n dueRetransform: false,\n earliestDeadlineAt: Math.max(\n earliestDeadlineAt,\n maintenanceNotBeforeAt,\n ),\n };\n }\n\n return {\n dueRevalidations: dueRevalidations.sort(compareByInsertionOrder),\n dueRetransform,\n earliestDeadlineAt,\n };\n }\n\n #removeConnection(\n selector: ConnectionSelector,\n revision?: number,\n ): Readonly<ConnectionContext> | undefined {\n const connection = this.#getConnectionContext(selector);\n\n if (!connection) {\n return undefined;\n }\n\n // If the revision has changed, we should not remove the connection\n if (revision !== undefined && connection.revision !== revision) {\n this.#lc.debug?.('Ignoring failConnection for stale revision', {\n clientID: selector.clientID,\n wsID: selector.wsID,\n attemptedRevision: revision,\n currentRevision: connection.revision,\n });\n return undefined;\n }\n\n this.#connections.delete(connection.clientID);\n this.#refreshBackgroundConnectionContext();\n this.#updateBackgroundRetransformDeadline(false);\n\n return connection;\n }\n\n #demoteConnection(connection: ConnectionContext): ConnectionContext {\n const demotedConnection = this.#storeConnection({\n ...connection,\n state: 'provisional',\n revalidateAt: undefined,\n });\n this.#refreshBackgroundConnectionContext();\n this.#updateBackgroundRetransformDeadline(false);\n return demotedConnection;\n }\n\n /**\n * Keeps the background connection sticky while it remains validated.\n *\n * When a newly validated `preferred` connection is provided, it is promoted\n * only if there is no current validated background connection. Otherwise the\n * existing background connection stays in place until it disappears or is\n * demoted, at which point the newest validated connection is selected.\n */\n #refreshBackgroundConnectionContext(preferred?: ConnectionContext): void {\n if (preferred?.state === 'validated') {\n const currentBackgroundConnection =\n this.#getBackgroundConnectionContext();\n if (\n currentBackgroundConnection?.clientID === preferred.clientID &&\n currentBackgroundConnection.wsID === preferred.wsID\n ) {\n return;\n }\n if (currentBackgroundConnection !== undefined) {\n return;\n }\n this.#setBackgroundConnection({\n clientID: preferred.clientID,\n wsID: preferred.wsID,\n });\n this.#lc.debug?.('Selected background connection for shared auth work', {\n clientID: preferred.clientID,\n wsID: preferred.wsID,\n revision: preferred.revision,\n reason: 'preferred-validated',\n });\n return;\n }\n\n const currentBackgroundConnection = this.#getBackgroundConnectionContext();\n if (currentBackgroundConnection?.state === 'validated') {\n return;\n }\n\n const nextBackgroundConnection = [...this.#connections.values()]\n .filter(connection => connection.state === 'validated')\n .sort(comparePreferredValidatedConnection)\n .at(0);\n this.#setBackgroundConnection(\n nextBackgroundConnection\n ? {\n clientID: nextBackgroundConnection.clientID,\n wsID: nextBackgroundConnection.wsID,\n }\n : undefined,\n );\n if (nextBackgroundConnection) {\n this.#lc.debug?.('Selected background connection for shared auth work', {\n clientID: nextBackgroundConnection.clientID,\n wsID: nextBackgroundConnection.wsID,\n revision: nextBackgroundConnection.revision,\n reason: 'fallback-validated',\n });\n }\n }\n\n #getBackgroundConnectionContext(): ConnectionContext | undefined {\n const backgroundConnection = this.#group.backgroundConnection;\n if (!backgroundConnection) {\n return undefined;\n }\n return this.#getConnectionContext(backgroundConnection);\n }\n\n #getConnectionContext(\n selector: ConnectionSelector,\n ): ConnectionContext | undefined {\n const connection = this.#connections.get(selector.clientID);\n if (!connection) {\n return undefined;\n }\n if (connection.wsID !== selector.wsID) {\n return undefined;\n }\n return connection;\n }\n\n #mustGetConnectionContext(selector: ConnectionSelector): ConnectionContext {\n const connection = this.#getConnectionContext(selector);\n\n if (!connection) {\n throw new ProtocolErrorWithLevel(\n {\n kind: ErrorKind.InvalidConnectionRequest,\n message:\n 'Connection auth state was not available for this websocket.',\n origin: ErrorOrigin.ZeroCache,\n },\n 'warn',\n );\n }\n\n return connection;\n }\n\n #storeConnection(connection: ConnectionContext): ConnectionContext {\n this.#connections.set(connection.clientID, connection);\n return connection;\n }\n\n #setGroup(group: GroupAuthState): GroupAuthState {\n this.#group = group;\n return group;\n }\n\n #setBackgroundConnection(\n backgroundConnection: ConnectionSelector | undefined,\n ) {\n if (\n sameConnectionSelector(\n this.#group.backgroundConnection,\n backgroundConnection,\n )\n ) {\n return;\n }\n this.#setGroup({\n ...this.#group,\n backgroundConnection: backgroundConnection\n ? {...backgroundConnection}\n : undefined,\n });\n }\n\n /**\n * Keeps the group background retransform deadline coherent with current\n * schedulability.\n *\n * When `reset` is false, this seeds a deadline only when shared retransform\n * is now possible and no deadline exists yet, preserving any existing\n * cadence. When `reset` is true, it starts a fresh interval from `#now()` if\n * retransform is schedulable for the current ready ViewSyncer instance, or\n * clears the deadline if it is not.\n */\n #updateBackgroundRetransformDeadline(reset: boolean) {\n const backgroundConnection = this.#getBackgroundConnectionContext();\n if (\n !backgroundConnection ||\n this.#retransformIntervalMs === undefined ||\n !this.#sharedRetransformReady\n ) {\n if (this.#group.retransformAt !== undefined) {\n this.#setGroup({\n ...this.#group,\n retransformAt: undefined,\n });\n }\n return;\n }\n\n if (reset || this.#group.retransformAt === undefined) {\n this.#setGroup({\n ...this.#group,\n retransformAt: this.#now() + this.#retransformIntervalMs,\n });\n }\n }\n\n #nextRevalidateAt() {\n return this.#revalidateIntervalMs === undefined\n ? undefined\n : this.#now() + this.#revalidateIntervalMs;\n }\n}\n\nfunction compareByInsertionOrder(\n a: Pick<ConnectionContext, 'insertionOrder' | 'wsID'>,\n b: Pick<ConnectionContext, 'insertionOrder' | 'wsID'>,\n) {\n return a.insertionOrder - b.insertionOrder || a.wsID.localeCompare(b.wsID);\n}\n\nfunction comparePreferredValidatedConnection(\n a: Pick<ConnectionContext, 'insertionOrder' | 'wsID'>,\n b: Pick<ConnectionContext, 'insertionOrder' | 'wsID'>,\n) {\n return b.insertionOrder - a.insertionOrder || b.wsID.localeCompare(a.wsID);\n}\n\nfunction minDefined(a: number | undefined, b: number | undefined) {\n if (a === undefined) {\n return b;\n }\n if (b === undefined) {\n return a;\n }\n return Math.min(a, b);\n}\n\nfunction sameConnectionSelector(\n a: ConnectionSelector | undefined,\n b: ConnectionSelector | undefined,\n) {\n return a?.clientID === b?.clientID && a?.wsID === b?.wsID;\n}\n\nfunction filterHeaders(\n headers: Readonly<Record<string, string>> | undefined,\n allowedHeaders: readonly string[] | undefined,\n) {\n if (!headers || !allowedHeaders || allowedHeaders.length === 0) {\n return undefined;\n }\n\n const allowed = new Set(allowedHeaders.map(header => header.toLowerCase()));\n let filtered: Record<string, string> | undefined;\n for (const [key, value] of Object.entries(headers)) {\n if (allowed.has(key.toLowerCase())) {\n filtered ??= {};\n filtered[key] = value;\n }\n }\n return filtered;\n}\n"],"mappings":";;;;;;;;;;;;;;AAiLA,IAAa,+BAAb,MAA8E;CAC5E;CAGA,+BAAwB,IAAI,IAA+B;CAC3D,SAAyB;EACvB,YAAY,KAAA;EACZ,sBAAsB,KAAA;EACtB,eAAe,KAAA;EACf,wBAAwB,KAAA;CAC1B;CAEA;CAEA;CACA;CACA;CACA;CACA;CACA,0BAA0B;CAC1B,sBAAsB;CAEtB,YACE,IACA,2BACA,4BACA,aACA,YACA,mBACA,KACA;EACA,KAAKA,MAAM;EACX,KAAKG,OAAO,OAAO,KAAK;EACxB,KAAKC,wBACH,8BAA8B,KAAA,IAC1B,KAAA,IACA,4BAA4B;EAClC,KAAKC,yBACH,+BAA+B,KAAA,IAC3B,KAAA,IACA,6BAA6B;EACnC,KAAKC,eAAe;EACpB,KAAKC,cAAc;EACnB,KAAKL,qBAAqB;CAC5B;;;;;;;CAQA,mBACE,UACA,eACA,MAC6B;EAC7B,KAAKM,kBAAkB,QAAQ;EAE/B,MAAM,cAAc,SAAqD;GACvE,MAAM,SAAS,SAAS,UAAU,KAAKF,eAAe,KAAKC;GAE3D,OAAO;IACL,KAAK,QAAQ,MAAM;IACnB,oBAAoB,QAAQ,KAAK,IAAI,iBAAiB;IACtD,eAAe;KACb,eAAe,KAAA;KACf,gBAAgB,cACd,cAAc,gBACd,QAAQ,qBACV;KACA,QAAQ,cAAc;KACtB,QAAQ,QAAQ;KAChB,QAAQ,QAAQ,iBAAiB,cAAc,aAAa,KAAA;IAC9D;GACF;EACF;EAEA,MAAM,aAAgC;GACpC,OAAO;GAEP,UAAU,cAAc;GACxB,MAAM,cAAc;GACpB,UAAU;GACV,MAAM,EAAC,IAAI,cAAc,UAAU,KAAI;GACvC;GAEA,WAAW,cAAc;GACzB,YAAY,cAAc;GAC1B,iBAAiB,cAAc;GAC/B,eAAe,cAAc;GAE7B,cAAc,KAAA;GAEd,cAAc,WAAW,OAAO;GAChC,eAAe,WAAW,QAAQ;GAElC,gBAAgB,EAAE,KAAKE;EACzB;EACA,KAAKC,iBAAiB,UAAU;EAChC,KAAKC,oCAAoC;EACzC,KAAKC,qCAAqC,KAAK;EAC/C,OAAO;CACT;;;;;;;CAQA,eACE,UACA,MAC6B;EAC7B,MAAM,aAAa,KAAKC,0BAA0B,QAAQ;EAE1D,IAAI,eAAe,WAAW;EAC9B,IAAI,gBAAgB,WAAW;EAE/B,IAAI,KAAK,cACP,eAAe;GACb,GAAG;GACH,KAAK,KAAK;EACZ;EAEF,IAAI,KAAK,kBACP,eAAe;GACb,GAAG;GACH,eAAe;IACb,GAAG,aAAa;IAChB,eAAe,cACb,KAAK,kBACL,KAAKP,cAAc,oBACrB;GACF;EACF;EAEF,IAAI,KAAK,aACP,gBAAgB;GACd,GAAG;GACH,KAAK,KAAK;EACZ;EAEF,IAAI,KAAK,iBACP,gBAAgB;GACd,GAAG;GACH,eAAe;IACb,GAAG,cAAc;IACjB,eAAe,cACb,KAAK,iBACL,KAAKC,aAAa,oBACpB;GACF;EACF;EAGF,OAAO,KAAKO,kBAAkB;GAC5B,GAAG;GACH,UAAU,WAAW,WAAW;GAChC;GACA;EACF,CAAC;CACH;;;;;CAMA,MAAM,WACJ,UACA,MACsC;EACtC,MAAM,aAAa,KAAKD,0BAA0B,QAAQ;EAE1D,MAAM,WAAW,MAAM,YACrB,KAAKb,KACL,WAAW,MACX,WAAW,KAAK,IAChB,KAAK,MACL,KAAKE,kBACP;EAGA,IAAI,CADiB,WAAW,WAAW,MAAM,QAAQ,GAEvD,OAAO,KAAKY,kBAAkB;GAC5B,GAAG;GACH,MAAM;GACN,UAAU,WAAW,WAAW;EAClC,CAAC;EAGH,IAAI,aAAa,WAAW,MAC1B,OAAO;EAGT,OAAO,KAAKJ,iBAAiB;GAC3B,GAAG;GACH,MAAM;EACR,CAAC;CACH;;;;;;;;;;CAWA,mBACE,UACA,UACA,YAMY;EACZ,MAAM,aAAa,KAAKK,sBAAsB,QAAQ;EACtD,IAAI,CAAC,YACH;EAGF,IAAI,WAAW,aAAa,UAAU;GACpC,KAAKf,IAAI,QAAQ,kDAAkD;IACjE,UAAU,SAAS;IACnB,mBAAmB;IACnB,iBAAiB,WAAW;GAC9B,CAAC;GACD;EACF;EAEA,IAAI;EAIJ,IAAI,WAAW,SAAS,oBAAoB;GAC1C,qBAAqB,EAAC,IAAI,WAAW,gBAAe;GAIpD,IAAI,WAAW,KAAK,OAAO,mBAAmB,IAC5C,MAAM,IAAI,uBACR;IACE,MAAM;IACN,SACE;IACF,QAAQ;GACV,GACA,MACF;EAEJ;EAIA,MAAM,oBAAoB,sBAAsB,WAAW;EAI3D,IACE,KAAKgB,OAAO,eAAe,KAAA,KAC3B,KAAKA,OAAO,WAAW,OAAO,kBAAkB,IAEhD,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SACE;GACF,QAAQ;EACV,GACA,MACF;EAGF,IAAI,KAAKA,OAAO,eAAe,KAAA,GAC7B,KAAKC,UAAU;GACb,GAAG,KAAKD;GACR,YAAY;EACd,CAAC;EAGH,MAAM,sBAAsB,KAAKN,iBAAiB;GAChD,GAAG;GACH,OAAO;GACP,cAAc,KAAKQ,kBAAkB;EACvC,CAAC;EACD,KAAKP,oCAAoC,mBAAmB;EAC5D,KAAKC,qCAAqC,KAAK;EAE/C,OAAO;GACL,YAAY;GACZ,OAAO,KAAK,cAAc;EAC5B;CACF;;CAGA,eACE,UACA,UAC+B;EAC/B,OAAO,KAAKJ,kBAAkB,UAAU,QAAQ;CAClD;;CAGA,gBAAgB,UAA6D;EAC3E,OAAO,KAAKA,kBAAkB,QAAQ;CACxC;;;;;;CAOA,iCACE,UACA,UACM;EACN,MAAM,uBAAuB,KAAKW,gCAAgC;EAClE,IAAI,CAAC,sBACH;EAEF,IACE,qBAAqB,aAAa,SAAS,YAC3C,qBAAqB,SAAS,SAAS,QACvC,qBAAqB,aAAa,UAElC;EAEF,KAAKP,qCAAqC,IAAI;CAChD;CAEA,0BAA0B,OAAsB;EAC9C,IAAI,KAAKQ,4BAA4B,OACnC;EAEF,KAAKA,0BAA0B;EAC/B,KAAKR,qCAAqC,IAAI;CAChD;CAEA,iBAAiB,MAA0C;EACzD,MAAM,aACJ,SAAS,eACL,KAAKR,wBACL,KAAKC;EACX,IAAI,eAAe,KAAA,GACjB;EAEF,KAAKY,UAAU;GACb,GAAG,KAAKD;GACR,wBAAwB,KAAK,IAC3B,KAAKA,OAAO,0BAA0B,GACtC,KAAKb,KAAK,IAAI,UAChB;EACF,CAAC;CACH;;CAGA,qBACE,UACyC;EACzC,OAAO,KAAKY,sBAAsB,QAAQ;CAC5C;;CAGA,yBACE,UAC6B;EAC7B,OAAO,KAAKF,0BAA0B,QAAQ;CAChD;;CAGA,iCAA0E;EACxE,OAAO,KAAKM,gCAAgC;CAC9C;CAEA,qCAAkE;EAChE,MAAM,uBAAuB,KAAKA,gCAAgC;EAClE,IAAI,CAAC,sBACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SACE;GACF,QAAQ;EACV,GACA,MACF;EAEF,OAAO;CACT;;CAGA,gBAA0C;EACxC,OAAO,KAAKH;CACd;;;;;;;;;CAUA,kBAIE;EACA,MAAM,mBAAkD,CAAC;EACzD,MAAM,MAAM,KAAKb,KAAK;EACtB,IAAI,qBAAqB,KAAKa,OAAO;EAErC,KAAK,MAAM,cAAc,KAAKf,aAAa,OAAO,GAAG;GACnD,IACE,WAAW,UAAU,eACrB,WAAW,iBAAiB,KAAA,GAE5B;GAEF,IAAI,WAAW,gBAAgB,KAC7B,iBAAiB,KAAK,UAAU;GAElC,qBAAqB,WACnB,oBACA,WAAW,YACb;EACF;EAEA,MAAM,iBACJ,KAAKe,OAAO,kBAAkB,KAAA,KAC9B,KAAKA,OAAO,iBAAiB;EAC/B,MAAM,yBAAyB,KAAKA,OAAO;EAE3C,IACE,2BAA2B,KAAA,KAC3B,yBAAyB,OACzB,uBAAuB,KAAA,GAEvB,OAAO;GACL,kBAAkB,CAAC;GACnB,gBAAgB;GAChB,oBAAoB,KAAK,IACvB,oBACA,sBACF;EACF;EAGF,OAAO;GACL,kBAAkB,iBAAiB,KAAK,uBAAuB;GAC/D;GACA;EACF;CACF;CAEA,kBACE,UACA,UACyC;EACzC,MAAM,aAAa,KAAKD,sBAAsB,QAAQ;EAEtD,IAAI,CAAC,YACH;EAIF,IAAI,aAAa,KAAA,KAAa,WAAW,aAAa,UAAU;GAC9D,KAAKf,IAAI,QAAQ,8CAA8C;IAC7D,UAAU,SAAS;IACnB,MAAM,SAAS;IACf,mBAAmB;IACnB,iBAAiB,WAAW;GAC9B,CAAC;GACD;EACF;EAEA,KAAKC,aAAa,OAAO,WAAW,QAAQ;EAC5C,KAAKU,oCAAoC;EACzC,KAAKC,qCAAqC,KAAK;EAE/C,OAAO;CACT;CAEA,kBAAkB,YAAkD;EAClE,MAAM,oBAAoB,KAAKF,iBAAiB;GAC9C,GAAG;GACH,OAAO;GACP,cAAc,KAAA;EAChB,CAAC;EACD,KAAKC,oCAAoC;EACzC,KAAKC,qCAAqC,KAAK;EAC/C,OAAO;CACT;;;;;;;;;CAUA,oCAAoC,WAAqC;EACvE,IAAI,WAAW,UAAU,aAAa;GACpC,MAAM,8BACJ,KAAKO,gCAAgC;GACvC,IACE,6BAA6B,aAAa,UAAU,YACpD,4BAA4B,SAAS,UAAU,MAE/C;GAEF,IAAI,gCAAgC,KAAA,GAClC;GAEF,KAAKE,yBAAyB;IAC5B,UAAU,UAAU;IACpB,MAAM,UAAU;GAClB,CAAC;GACD,KAAKrB,IAAI,QAAQ,uDAAuD;IACtE,UAAU,UAAU;IACpB,MAAM,UAAU;IAChB,UAAU,UAAU;IACpB,QAAQ;GACV,CAAC;GACD;EACF;EAGA,IADoC,KAAKmB,gCACrC,GAA6B,UAAU,aACzC;EAGF,MAAM,2BAA2B,CAAC,GAAG,KAAKlB,aAAa,OAAO,CAAC,EAC5D,QAAO,eAAc,WAAW,UAAU,WAAW,EACrD,KAAK,mCAAmC,EACxC,GAAG,CAAC;EACP,KAAKoB,yBACH,2BACI;GACE,UAAU,yBAAyB;GACnC,MAAM,yBAAyB;EACjC,IACA,KAAA,CACN;EACA,IAAI,0BACF,KAAKrB,IAAI,QAAQ,uDAAuD;GACtE,UAAU,yBAAyB;GACnC,MAAM,yBAAyB;GAC/B,UAAU,yBAAyB;GACnC,QAAQ;EACV,CAAC;CAEL;CAEA,kCAAiE;EAC/D,MAAM,uBAAuB,KAAKgB,OAAO;EACzC,IAAI,CAAC,sBACH;EAEF,OAAO,KAAKD,sBAAsB,oBAAoB;CACxD;CAEA,sBACE,UAC+B;EAC/B,MAAM,aAAa,KAAKd,aAAa,IAAI,SAAS,QAAQ;EAC1D,IAAI,CAAC,YACH;EAEF,IAAI,WAAW,SAAS,SAAS,MAC/B;EAEF,OAAO;CACT;CAEA,0BAA0B,UAAiD;EACzE,MAAM,aAAa,KAAKc,sBAAsB,QAAQ;EAEtD,IAAI,CAAC,YACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SACE;GACF,QAAQ;EACV,GACA,MACF;EAGF,OAAO;CACT;CAEA,iBAAiB,YAAkD;EACjE,KAAKd,aAAa,IAAI,WAAW,UAAU,UAAU;EACrD,OAAO;CACT;CAEA,UAAU,OAAuC;EAC/C,KAAKe,SAAS;EACd,OAAO;CACT;CAEA,yBACE,sBACA;EACA,IACE,uBACE,KAAKA,OAAO,sBACZ,oBACF,GAEA;EAEF,KAAKC,UAAU;GACb,GAAG,KAAKD;GACR,sBAAsB,uBAClB,EAAC,GAAG,qBAAoB,IACxB,KAAA;EACN,CAAC;CACH;;;;;;;;;;;CAYA,qCAAqC,OAAgB;EAEnD,IACE,CAF2B,KAAKG,gCAE/B,KACD,KAAKd,2BAA2B,KAAA,KAChC,CAAC,KAAKe,yBACN;GACA,IAAI,KAAKJ,OAAO,kBAAkB,KAAA,GAChC,KAAKC,UAAU;IACb,GAAG,KAAKD;IACR,eAAe,KAAA;GACjB,CAAC;GAEH;EACF;EAEA,IAAI,SAAS,KAAKA,OAAO,kBAAkB,KAAA,GACzC,KAAKC,UAAU;GACb,GAAG,KAAKD;GACR,eAAe,KAAKb,KAAK,IAAI,KAAKE;EACpC,CAAC;CAEL;CAEA,oBAAoB;EAClB,OAAO,KAAKD,0BAA0B,KAAA,IAClC,KAAA,IACA,KAAKD,KAAK,IAAI,KAAKC;CACzB;AACF;AAEA,SAAS,wBACP,GACA,GACA;CACA,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,KAAK,cAAc,EAAE,IAAI;AAC3E;AAEA,SAAS,oCACP,GACA,GACA;CACA,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,KAAK,cAAc,EAAE,IAAI;AAC3E;AAEA,SAAS,WAAW,GAAuB,GAAuB;CAChE,IAAI,MAAM,KAAA,GACR,OAAO;CAET,IAAI,MAAM,KAAA,GACR,OAAO;CAET,OAAO,KAAK,IAAI,GAAG,CAAC;AACtB;AAEA,SAAS,uBACP,GACA,GACA;CACA,OAAO,GAAG,aAAa,GAAG,YAAY,GAAG,SAAS,GAAG;AACvD;AAEA,SAAS,cACP,SACA,gBACA;CACA,IAAI,CAAC,WAAW,CAAC,kBAAkB,eAAe,WAAW,GAC3D;CAGF,MAAM,UAAU,IAAI,IAAI,eAAe,KAAI,WAAU,OAAO,YAAY,CAAC,CAAC;CAC1E,IAAI;CACJ,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,GAC/C,IAAI,QAAQ,IAAI,IAAI,YAAY,CAAC,GAAG;EAClC,aAAa,CAAC;EACd,SAAS,OAAO;CAClB;CAEF,OAAO;AACT"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"inspect-handler.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/inspect-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,uDAAuD,CAAC;
|
|
1
|
+
{"version":3,"file":"inspect-handler.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/inspect-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,uDAAuD,CAAC;AAC9F,OAAO,KAAK,EACV,kBAAkB,EAEnB,MAAM,+CAA+C,CAAC;AACvD,OAAO,EAEL,KAAK,qBAAqB,EAC3B,MAAM,6CAA6C,CAAC;AAIrD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAMpE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAE1E,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,iCAAiC,CAAC;AACvE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAC7C,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,UAAU,CAAC;AAE1C,wBAAsB,aAAa,CACjC,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,qBAAqB,EAC3B,GAAG,EAAE,WAAW,EAChB,MAAM,EAAE,aAAa,EACrB,iBAAiB,EAAE,iBAAiB,EACpC,aAAa,EAAE,MAAM,EAGrB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,oBAAoB,EAC5B,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,IAAI,CAAC,CAmJf;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,kBAAkB,GACzB,kBAAkB,CAUpB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,kBAAkB,GAAG,IAAI,EAClC,eAAe,EAAE,MAAM,GACtB,kBAAkB,GAAG,IAAI,CAmB3B"}
|
|
@@ -85,7 +85,7 @@ async function handleInspect(lc, body, cvr, client, inspectorDelegate, clientGro
|
|
|
85
85
|
client.sendInspectResponse(lc, {
|
|
86
86
|
op: "analyze-query",
|
|
87
87
|
id: body.id,
|
|
88
|
-
value: ctx.
|
|
88
|
+
value: ctx.protocolFlags.has(1) ? result : withoutFilterNodes(result)
|
|
89
89
|
});
|
|
90
90
|
break;
|
|
91
91
|
}
|