npm - @rocicorp/zero - Versions diffs - 1.12.0-canary.6 → 1.12.0-canary.8 - Mend

@rocicorp/zero 1.12.0-canary.6 → 1.12.0-canary.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (379) hide show

package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport type {ReplicaConstraints} from '../../litestream/commands.ts';\nimport {\n getSubscriptionStateAndContext,\n type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n restoreReplica,\n type ConstrainingPurgeLock,\n type InitializeResult,\n type InitializeTimings,\n type RestoreOptions,\n type ResumableBackup,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n initialSync,\n type InitialSyncOptions,\n type ReplicaOptions,\n type ServerContext,\n} from './initial-sync.ts';\nimport {endSession} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n claimSlotForResumption,\n createReplicaAndSlot,\n type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n dropShard,\n getActiveReplicas,\n getReplicaAtVersion,\n getRestoreCandidates,\n initRestoreReplica,\n internalPublicationPrefix,\n type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n // Create a new slot for the replica rather than taking over\n // an existing one (i.e. high-availability mode).\n slotPerReplica: boolean \| undefined;\n\n inactiveReplicaGracePeriodMs: number;\n\n /*\n Whether this replication-manager maintains the PG change log, i.e. acks\n * the replication slot only after transactions are stored in it. Recorded\n * with the replica (see `pgChangeLog` in `ReplicaState`) when its slot is\n * taken over.\n /\n pgChangeLog: boolean;\n};\n\n/\n Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n /\nexport async function initializePostgresChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n syncOptions: InitialSyncOptions,\n context: ServerContext,\n lagReportIntervalMs = 0,\n restoreOptions: RestoreOptions = {},\n {\n epoch,\n slotPerReplica,\n backupV5,\n inactiveReplicaGracePeriodMs,\n pgChangeLog,\n }: InitializeOptions = {\n epoch: 0,\n slotPerReplica: false,\n backupV5: true,\n inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n pgChangeLog: false,\n },\n streamInboundTimeoutMs?: number \| undefined,\n): Promise<InitializeResult> {\n const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n try {\n await ensureShardSchema(\n lc,\n db,\n shard,\n syncOptions.installPartialIndexTriggers,\n );\n\n const {\n replica: restoredReplica,\n purgeLock,\n pgChangeLogBehindSlot,\n resumableBackup,\n slotWatermark,\n timings,\n } = slotPerReplica\n ? await forkOrResumeReplica(\n lc,\n db,\n shard,\n epoch,\n syncOptions.replicationSlotFailover ?? false,\n replicaDbFile,\n restoreOptions,\n inactiveReplicaGracePeriodMs,\n )\n : await selectAndRestoreReplica(\n lc,\n db,\n shard,\n replicaDbFile,\n restoreOptions,\n );\n\n let initialSyncedReplica: ReplicaState \| undefined;\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n async (log, tx) => {\n // Falling back to initial sync abandons the restore, so release the\n // resources acquired for it, e.g. a replication slot claimed (or\n // created) for a resumed (or forked) replica, which would otherwise\n // remain active indefinitely.\n await restoreOptions.cleanup?.release();\n // The purge lock on the change-db must be released before performing\n // initial sync; if the change-db and upstream are the same db, a lock-holding\n // transaction will prevent a replication slot from being created. This awkward\n // dependency can go away with RMv2. (The purge lock is also released by\n // the cleanup when it is registered with it; releasing it is idempotent.)\n await purgeLock?.release();\n initialSyncedReplica = await initialSync(\n log,\n shard,\n tx,\n upstreamURI,\n syncOptions,\n context,\n {epoch, backupV5, pgChangeLog},\n restoreOptions.cleanup,\n );\n },\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionStateAndContext(\n new StatementRunner(replica),\n );\n replica.close();\n\n // Check that upstream is properly setup, and throw an AutoReset to re-run\n // initial sync if not.\n const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n lc,\n db,\n shard,\n subscriptionState,\n (initialSyncedReplica ?? restoredReplica)?.id,\n );\n\n // With litestream v5, every replication-manager backs up to its own\n // backup lineage.\n const newBackupLineage = backupV5 && !resumableBackup;\n const backupPath = initialSyncedReplica\n ? // If initial sync was performed, use that initial backupPath.\n initialSyncedReplica.backupPath\n : // Otherwise, use the lineage prepared by the restore, or a new, unique\n // path when backing up with litestream v5. This will be recorded in the\n // replicas table by the PostgresChangeSource.\n backupV5\n ? (resumableBackup?.backupPath ?? String(Date.now()))\n : (restoredReplica?.backupPath ?? null);\n\n const changeSource = new PostgresChangeSource(\n lc,\n upstreamURI,\n shard,\n upstreamReplica,\n pgVersion,\n {backupPath, backupV5, pgChangeLog},\n context,\n lagReportIntervalMs,\n syncOptions.textCopy,\n streamInboundTimeoutMs,\n minStartWatermark(\n subscriptionState.watermark,\n initialSyncedReplica !== undefined,\n slotWatermark,\n purgeLock,\n ),\n );\n\n const destinationBackupURL =\n backupPath && restoreOptions.litestream?.backupURL\n ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n restoreOptions.litestream?.backupURL;\n\n return {\n subscriptionState,\n changeSource,\n destinationBackupURL,\n // The replica this change stream belongs to. It is part of the identity\n // the SQLite change log records, because a generation (i.e.\n // `replicaVersion`) is shared by every sibling of a forked replica and so\n // cannot distinguish two siblings' logs.\n replicaID: upstreamReplica.id,\n waitForBackupBeforeServing:\n // Wait for the first backup if there was an initial sync,\n initialSyncedReplica !== undefined \|\|\n // or if the destination differs from where it was restored\n // (i.e. backupV5).\n backupPath !== (restoredReplica?.backupPath ?? null),\n pgChangeLogBehindSlot,\n newBackupLineage,\n startTimings: initialSyncedReplica\n ? {kind: 'initial_sync'}\n : (timings ?? {kind: slotPerReplica ? 'fork' : 'rmv1'}),\n };\n } finally {\n await db.end();\n }\n}\n\ntype RestoredReplica = {\n replica: ReplicaState \| undefined;\n purgeLock: ConstrainingPurgeLock \| null;\n pgChangeLogBehindSlot?: boolean \| undefined;\n resumableBackup?: ResumableBackup \| undefined;\n /\n The position (as a watermark) of the replication slot from which\n * replication resumes, when it was created (for a fork) or claimed (for a\n * resumption), i.e. while it is held by this process. Absent for a shared\n * slot (RMv1), whose position is advanced by its current owner until it is\n * taken over, and for a resumed slot that was acked only once transactions\n * were stored in the PG change-log (RMv1.5), whose position may be past the\n * change-log's head for acked keepalives.\n /\n slotWatermark?: string \| undefined;\n timings?: InitializeTimings \| undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n// intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n replicaFile: string,\n {litestream, acquirePurgeLock}: RestoreOptions,\n): Promise<RestoredReplica> {\n // The purge lock constrains the generation of the replica to restore. With\n // a shared replication slot, the change-log is resumed from its head\n // without being checked against the slot's position: the slot is still\n // being acked by the previous replication-manager, on an idle upstream past\n // the change-log's head (see PgChangeLogPurgeLocker).\n const lock = await acquirePurgeLock?.();\n assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n const purgeLock = lock ?? null;\n const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n // filter to the generation specified by the purge lock, if present\n ({generation}) => generation === (purgeLock?.replicaVersion ?? generation),\n );\n if (replicas.length === 0) {\n lc.info?.(`no suitable replicas to restore from`, {replicas});\n return {replica: undefined, purgeLock};\n }\n const [replica] = replicas;\n\n if (litestream?.backupURL) {\n const constraints = restoreConstraints(purgeLock, replica);\n const {backupURL: backupBaseURL} = litestream;\n const {slot, backupPath, confirmedFlushLsn} = replica;\n const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n lc.info?.(\n `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n {replicas},\n );\n const restoreStart = performance.now();\n const resumableBackup = await restoreReplica(\n lc,\n {...litestream, backupURL}, // includes the replica's backup sub-path\n replicaFile,\n constraints,\n litestream.forkBackup\n ? {type: 'fork', baseURL: backupBaseURL}\n : undefined,\n );\n const restoreMs = performance.now() - restoreStart;\n return {\n replica,\n purgeLock,\n resumableBackup,\n timings: {kind: 'rmv1', restoreMs},\n };\n }\n return {replica, purgeLock, timings: {kind: 'rmv1'}};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n epoch: number,\n slotFailover: boolean,\n replicaFile: string,\n {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n gracePeriodMs: number,\n): Promise<RestoredReplica> {\n const forkWaitStart = performance.now();\n const result = await getSourceAndDestinationReplicas(\n lc,\n sql,\n shard,\n epoch,\n slotFailover,\n gracePeriodMs,\n // The PG change-log is checked against the slot's position (see below)\n // only when this replication-manager maintains it.\n acquirePurgeLock !== undefined,\n );\n if (!result) {\n // can't restore, must initial-sync\n return {replica: undefined, purgeLock: null};\n }\n const {restoreFrom, replicateTo, slotLSN, slotSession} = result;\n const timings: InitializeTimings = {\n kind: restoreFrom.id === replicateTo.id ? 'resume' : 'fork',\n forkWaitMs: performance.now() - forkWaitStart,\n };\n // A resumed slot (i.e. of the same replica) that was acked only once\n // transactions were stored in the PG change-log (RMv1.5) cannot have been\n // acked past a transaction missing from the change-log. Its position past\n // the change-log's head only reflects acked keepalives, so it is not\n // compared with the head (see below).\n const resumedAfterPgChangeLog =\n restoreFrom.id === replicateTo.id && restoreFrom.pgChangeLog;\n const slotWatermark = resumedAfterPgChangeLog\n ? undefined\n : toStateVersionString(slotLSN);\n // The slot is held active by a session in this process until it is taken\n // over by the PostgresChangeSource. If initialization fails before then,\n // the session must be released, or the slot remains active indefinitely,\n // preventing the replica from being resumed (or forked) by a retry.\n cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () =>\n slotSession.destroy(),\n );\n\n // Acquired only after the replication slot has been created (or claimed),\n // and only if the change-log can be resumed from the slot's position.\n //\n // For a fork, this is the case whenever the change-log is maintained (i.e.\n // by a replication-manager that acks the slot only once a transaction has\n // been stored in the change-log): the fork waits for the source replica's\n // slot to pass the position after the creation of the new replica, which is\n // a transaction (in the publication) after the new slot's position (see\n // `waitForStoredTxPastSlot` in getSourceAndDestinationReplicas()).\n //\n // For a resumed slot, the slot can also have been acked past the head of a\n // maintained change-log, for keepalives (e.g. for changes outside of the\n // publication, or on an idle upstream). If the slot was acked by a\n // replication-manager that maintains the change-log, it is therefore not\n // checked against the slot's position, which would otherwise re-initialize\n // the change-log unnecessarily, resetting subscribers behind the replica.\n // Otherwise (e.g. RMv2, or unknown), it is: the change-log is then\n // re-initialized from the replica if it does not contain the replica's\n // changes.\n const lock = acquirePurgeLock ? await acquirePurgeLock(slotWatermark) : null;\n const pgChangeLogBehindSlot = lock === 'behind-slot';\n const purgeLock = lock === 'behind-slot' ? null : lock;\n // Absent a purge lock, still constrain the restore to the source's\n // generation, so that a local replica left behind by a different\n // generation (e.g. a failed initial sync) is not mistaken for it.\n const constraints = restoreConstraints(purgeLock, restoreFrom);\n\n if (litestream?.backupURL) {\n const {backupURL: backupBaseURL} = litestream;\n const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n lc.info?.(\n `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n {restoreFrom, replicateTo},\n );\n const restoreStart = performance.now();\n const resumableBackup = await restoreReplica(\n lc,\n {...litestream, backupURL}, // includes the replica's backup sub-path\n replicaFile,\n constraints,\n litestream.forkBackup\n ? {type: 'fork', baseURL: backupBaseURL}\n : undefined,\n );\n timings.restoreMs = performance.now() - restoreStart;\n return {\n replica: replicateTo,\n purgeLock,\n pgChangeLogBehindSlot,\n resumableBackup,\n slotWatermark,\n timings,\n };\n }\n return {\n replica: replicateTo,\n purgeLock,\n pgChangeLogBehindSlot,\n slotWatermark,\n timings,\n };\n}\n\n/\n The watermark below which the replication stream cannot (correctly) be\n * started, i.e. from which it would skip transactions that the change log\n * does not contain. It reflects the order in which each mode stores a\n * transaction and then acks it to the slot, such that a correctly initialized\n * stream always starts at or past it:\n \n * After an initial sync, the replica's watermark, which is the position\n * at which the new slot was created, and the change log is (re)initialized\n * at it.\n * * With a slot per replica (RMv1.5 and RMv2), the minimum of the restored\n * replica's watermark and the slot's position. Transactions are only acked\n * once backed up (so a skipped interval ends at or before the replica), and\n * if the change log is behind the slot it is re-initialized from the\n * replica. Note that the slot can be past the replica (acked for keepalives,\n * with nothing skipped), and the replica past the change log (forwarded\n * before being stored, and streamed again from the slot), but the stream\n * never starts below both. (With RMv1.5's purge lock, the change log's head\n * is at or past the slot, so it does not lower the minimum.)\n * * With a shared slot (RMv1), or a resumed slot that was acked only once\n * transactions were stored in the change log (RMv1.5), the change log's\n * head when it was purge locked, as the stream resumes from the head, which\n * only advances. The slot is not a reference: a shared slot is acked by its\n * owner until it is taken over, and a resumed RMv1.5 slot can be past the\n * head for acked keepalives. Without a purge lock (i.e. an empty change\n * log), there is no floor.\n /\n// Exported for testing.\nexport function minStartWatermark(\n replicaWatermark: string,\n initialSynced: boolean,\n slotWatermark: string \| undefined,\n purgeLock: ConstrainingPurgeLock \| null,\n): string \| undefined {\n if (initialSynced) {\n return replicaWatermark;\n }\n if (slotWatermark !== undefined) {\n return replicaWatermark < slotWatermark ? replicaWatermark : slotWatermark;\n }\n return purgeLock?.headWatermark;\n}\n\nfunction restoreConstraints(\n purgeLock: ConstrainingPurgeLock \| null,\n {generation}: ReplicaState,\n): ReplicaConstraints {\n return purgeLock ?? {replicaVersion: generation, minWatermark: ''};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 500;\n// Polling is frequent so that a fork proceeds as soon as the source replica\n// passes the fork LSN; logging is throttled by time rather than per poll.\nconst PROGRESS_LOG_INTERVAL_MS = 5_000;\nconst SLOW_PROGRESS_LOG_INTERVAL_MS = 60_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n epoch: number,\n slotFailover: boolean,\n gracePeriodMs: number,\n /\n Whether a fork waits for the source replica's slot to pass a transaction\n * after the destination slot's position (rather than just the position),\n * which is necessary for the PG change-log to be resumable from the\n * destination slot (see forkOrResumeReplica()).\n /\n waitForStoredTxPastSlot: boolean,\n pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n \| {\n restoreFrom: ReplicaState;\n replicateTo: ReplicaState;\n /* The position of the slot from which replication resumes. /\n slotLSN: LSN;\n /\n The session holding the destination slot active until it is taken\n * over by the PostgresChangeSource. Destroying it releases the slot.\n /\n slotSession: Readable;\n }\n \| undefined\n> {\n const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n let destination: ReplicationSlotResult<void> \| undefined;\n // The position that the source replica's slot must reach to be forked.\n let forkLSN: LSN \| undefined;\n const lastLogged = new Map<string, number>();\n const shouldLog = (key: string, intervalMs: number) => {\n const now = Date.now();\n const last = lastLogged.get(key);\n if (last !== undefined && now - last < intervalMs) {\n return false;\n }\n lastLogged.set(key, now);\n return true;\n };\n\n try {\n for (let i = 0; ; i++) {\n if (i > 0) {\n await sleep(pollIntervalMs);\n }\n const replicas = await getRestoreCandidates(sql, shard, epoch);\n if (shouldLog('replicas', PROGRESS_LOG_INTERVAL_MS)) {\n lc.info?.(`current replicas at epoch ${epoch}`, {replicas});\n }\n if (replicas.length === 0) {\n lc.info?.(`no suitable replicas to restore from`, {replicas});\n await endSession(destination?.initialSession);\n return undefined;\n }\n for (const replica of replicas) {\n // Track inactivity to resume replicas after a grace period.\n if (replica.active) {\n inactiveSince.delete(replica.id);\n } else if (!inactiveSince.has(replica.id)) {\n inactiveSince.set(replica.id, Date.now());\n }\n\n // Note: Only `active` InitialSync replicas are returned from\n // getRestoreCandidates().\n if (replica.stage === InitialSync) {\n // Log periodically; initial-sync can be long\n if (\n shouldLog(\n `initial-sync:${replica.id}`,\n SLOW_PROGRESS_LOG_INTERVAL_MS,\n )\n ) {\n lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n }\n break;\n }\n\n if (replica.stage === Replicate) {\n const {active, confirmedFlushLsn} = replica;\n if (active) {\n // Create a replication slot to fork the active replica.\n destination ??= await createReplicaAndSlot(\n lc,\n sql,\n 'fork-replica-session',\n shard,\n epoch,\n Date.now().toString(), // replicaID\n slotFailover,\n {\n backupPath: null, // set only after the backup has been confirmed\n backupV5: true, // RMv2 requires backupV5\n // The slot is not acked until it is taken over, which sets\n // this. Until then, it is unknown.\n pgChangeLog: false,\n },\n () => promiseVoid,\n ReplicaStage.Restore,\n );\n // The source replica acks its slot only once its changes are\n // backed up (and, if it maintains the PG change-log, stored), so\n // its slot reaching the destination slot's position means that\n // the destination can be restored from its backup.\n //\n // If the PG change-log is to be resumed from the destination slot,\n // the fork waits for the source to pass the position after the\n // creation of the destination replica instead. Its row is\n // inserted in a transaction (in the publication) after the\n // destination slot's position, so this ensures that a source\n // replica that maintains the PG change-log has stored a\n // transaction past the destination slot's position, rather than\n // only acked keepalives past it.\n //\n // Otherwise, the ack of that transaction (at its commit LSN, which\n // precedes the position after it) suffices, which the fork request\n // (see fork-request.ts) prompts the source to make promptly.\n forkLSN ??= waitForStoredTxPastSlot\n ? await currentWalLSN(sql)\n : destination.slot.consistent_point;\n if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n if (shouldLog(`fork:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)) {\n lc.info?.(\n `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n {replica},\n );\n }\n break;\n }\n }\n\n // If the replica is past the destination LSN, fork it, regardless of\n // whether its slot is still active. Forking is preferable to resuming,\n // as resuming removes the replica from being a candidate for\n // subsequent forks, and carries the risk of stealing the slot from a\n // task attempting to reconnect.\n if (\n destination &&\n forkLSN &&\n toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n ) {\n lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n replica,\n });\n const replicateTo = must(\n await initRestoreReplica(sql, shard, {\n sourceID: replica.id,\n destID: destination.replica.id,\n }),\n `replica ${destination.replica.id} was deleted`,\n );\n return {\n restoreFrom: replica,\n replicateTo,\n slotLSN: destination.slot.consistent_point,\n slotSession: destination.initialSession,\n };\n }\n }\n\n if (\n replica.stage === ReplicaStage.Restore &&\n replica.active &&\n replica.id !== destination?.replica.id\n ) {\n // Another task is restoring (e.g. resuming) this replica.\n if (\n shouldLog(`restoring:${replica.id}`, SLOW_PROGRESS_LOG_INTERVAL_MS)\n ) {\n lc.info?.(\n `waiting for replica ${replica.id}@${replica.slot} being restored by another task`,\n {replica},\n );\n }\n }\n\n if (replica.backupPath && !replica.active) {\n // An inactive replica that has a backupPath may be orphaned, or it\n // may be an active task that was temporarily disconnected and\n // attempting to reestablish a session.\n //\n // Resume the replica after a grace period to avoid stealing the slot\n // from an active task that was temporarily disconnected. Resumption\n // is a last resort only taken in the absence of alternatives.\n const now = Date.now();\n const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n if (\n inactiveMs >= gracePeriodMs \|\|\n shouldLog(`inactive:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)\n ) {\n lc.info?.(\n `replica ${replica.id}@${replica.slot} has been inactive for ${inactiveMs}ms`,\n {replica},\n );\n }\n if (inactiveMs >= gracePeriodMs) {\n const reserved = await claimSlotForResumption(\n lc,\n sql,\n shard,\n 'resume-replica',\n replica,\n );\n if (!reserved) {\n lc.warn?.(\n `unable to resume replica ${replica.id}@${replica.slot}`,\n );\n } else {\n lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n // If a new replication slot was created in anticipation of forking,\n // cancel it\n await endSession(destination?.initialSession);\n return {\n restoreFrom: replica,\n replicateTo: reserved.replica,\n // The slot is inactive, so its position is fixed.\n slotLSN: replica.confirmedFlushLsn,\n slotSession: reserved.reservation,\n };\n }\n }\n }\n }\n }\n } catch (e) {\n await endSession(destination?.initialSession);\n throw e;\n }\n}\n\nasync function checkAndUpdateUpstream(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n {\n replicaVersion,\n publications: subscribed,\n initialSyncContext,\n }: SubscriptionStateAndContext,\n replicaID: string \| undefined,\n) {\n const upstreamReplica = await getReplicaAtVersion(\n lc,\n sql,\n shard,\n replicaVersion,\n replicaID,\n initialSyncContext,\n );\n if (!upstreamReplica) {\n throw new AutoResetSignal(\n `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n );\n }\n\n // Verify that the publications match what is being replicated.\n const requested = shard.publications.toSorted();\n const replicated = upstreamReplica.publications\n .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n .sort();\n if (!deepEqual(requested, replicated)) {\n lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n throw new AutoResetSignal(\n `Requested publications [${requested}] do not match configured ` +\n `publications: [${replicated}]`,\n );\n }\n\n // Sanity check: The subscription state on the replica should have the\n // same publications. This should be guaranteed by the equivalence of the\n // replicaVersion, but it doesn't hurt to verify.\n if (!deepEqual(upstreamReplica.publications, subscribed)) {\n throw new AutoResetSignal(\n `Upstream publications [${upstreamReplica.publications}] do not ` +\n `match subscribed publications [${subscribed}]`,\n );\n }\n\n // Verify that the publications exist.\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n `.values();\n if (exists.length !== subscribed.length) {\n throw new AutoResetSignal(\n `Upstream publications [${exists.flat()}] do not contain ` +\n `all subscribed publications [${subscribed}]`,\n );\n }\n\n const {slot} = upstreamReplica;\n const result = await sql<{restartLSN: LSN \| null; walStatus: string \| null}[]>\n /sql/ `\n SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n WHERE slot_name = ${slot}`;\n if (result.length === 0) {\n throw new AutoResetSignal(`replication slot ${slot} is missing`);\n }\n const [{restartLSN, walStatus}] = result;\n if (restartLSN === null \|\| walStatus === 'lost') {\n throw new AutoResetSignal(\n `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n );\n }\n const [{pgVersion}] = await sql<{pgVersion: number}[]> /sql*/ `\n SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AA0EA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,8BACA,gBACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;CAC9B,aAAa;AACf,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,uBACA,iBACA,eACA,YACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,MAAM,eAAe,SAAS,QAAQ;GAMtC,MAAM,WAAW,QAAQ;GACzB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;IAAU;GAAW,GAC7B,eAAe,OACjB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,gBAAA,EAAkB,EAC7C;EAIA,MAAM,mBAAmB,YAAY,CAAC;EACtC,MAAM,aAAa,uBAEf,qBAAqB,aAIrB,WACG,iBAAiB,cAAc,OAAO,KAAK,IAAI,CAAC,IAChD,iBAAiB,cAAc;EA2BtC,OAAO;GACL;GACA,cAAA,IA3BuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;IAAU;GAAW,GAClC,SACA,qBACA,YAAY,UACZ,wBACA,kBACE,kBAAkB,WAClB,yBAAyB,KAAA,GACzB,eACA,SACF,CAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,CAAC,CAAC,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;GACA;GACA,cAAc,uBACV,EAAC,MAAM,eAAc,IACpB,WAAW,EAAC,MAAM,iBAAiB,SAAS,OAAM;EACzD;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAsBA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,oBACa;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,EAAA,CAAG,QAExD,EAAC,iBAAgB,gBAAgB,WAAW,kBAAkB,WACjE;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,cAAc,mBAAmB,WAAW,OAAO;EACzD,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,CAAC,CAAC,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EACA,MAAM,eAAe,YAAY,IAAI;EAWrC,OAAO;GACL;GACA;GACA,iBAAA,MAb4B,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,CACN;GAME,SAAS;IAAC,MAAM;IAAQ,WALR,YAAY,IAAI,IAAI;GAKH;EACnC;CACF;CACA,OAAO;EAAC;EAAS;EAAW,SAAS,EAAC,MAAM,OAAM;CAAC;AACrD;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,kBAAkB,WAC/B,eAC0B;CAC1B,MAAM,gBAAgB,YAAY,IAAI;CACtC,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,eAGA,qBAAqB,KAAA,CACvB;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,SAAS,gBAAe;CACzD,MAAM,UAA6B;EACjC,MAAM,YAAY,OAAO,YAAY,KAAK,WAAW;EACrD,YAAY,YAAY,IAAI,IAAI;CAClC;CAQA,MAAM,gBADJ,YAAY,OAAO,YAAY,MAAM,YAAY,cAE/C,KAAA,IACA,qBAAqB,OAAO;CAKhC,SAAS,UAAU,oBAAoB,YAAY,cACjD,YAAY,QAAQ,CACtB;CAqBA,MAAM,OAAO,mBAAmB,MAAM,iBAAiB,aAAa,IAAI;CACxE,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAIlD,MAAM,cAAc,mBAAmB,WAAW,WAAW;CAE7D,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,CAAC,CAAC,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EACA,MAAM,eAAe,YAAY,IAAI;EACrC,MAAM,kBAAkB,MAAM,eAC5B,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,aACA,WAAW,aACP;GAAC,MAAM;GAAQ,SAAS;EAAa,IACrC,KAAA,CACN;EACA,QAAQ,YAAY,YAAY,IAAI,IAAI;EACxC,OAAO;GACL,SAAS;GACT;GACA;GACA;GACA;GACA;EACF;CACF;CACA,OAAO;EACL,SAAS;EACT;EACA;EACA;EACA;CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,kBACd,kBACA,eACA,eACA,WACoB;CACpB,IAAI,eACF,OAAO;CAET,IAAI,kBAAkB,KAAA,GACpB,OAAO,mBAAmB,gBAAgB,mBAAmB;CAE/D,OAAO,WAAW;AACpB;AAEA,SAAS,mBACP,WACA,EAAC,cACmB;CACpB,OAAO,aAAa;EAAC,gBAAgB;EAAY,cAAc;CAAE;AACnE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AAGjC,IAAM,2BAA2B;AACjC,IAAM,gCAAgC;AACtC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eAOA,yBACA,iBAAiB,0BAcjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;CACJ,MAAM,6BAAa,IAAI,IAAoB;CAC3C,MAAM,aAAa,KAAa,eAAuB;EACrD,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,OAAO,WAAW,IAAI,GAAG;EAC/B,IAAI,SAAS,KAAA,KAAa,MAAM,OAAO,YACrC,OAAO;EAET,WAAW,IAAI,KAAK,GAAG;EACvB,OAAO;CACT;CAEA,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,KAAK,OAAO,KAAK;GAC7D,IAAI,UAAU,YAAY,wBAAwB,GAChD,GAAG,OAAO,6BAA6B,SAAS,EAAC,SAAQ,CAAC;GAE5D,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,MAAM,WAAW,aAAa,cAAc;IAC5C;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IACE,UACE,gBAAgB,QAAQ,MACxB,6BACF,GAEA,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,CAAC,CAAC,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;OAGV,aAAa;MACf,SACM,aACN,CACF;MAkBA,YAAY,0BACR,MAAM,cAAc,GAAG,IACvB,YAAY,KAAK;MACrB,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,IAAI,UAAU,QAAQ,QAAQ,MAAM,wBAAwB,GAC1D,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OAEF;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;OAC1B,aAAa,YAAY;MAC3B;KACF;IACF;IAEA,IACE,QAAQ,UAAU,KAClB,QAAQ,UACR,QAAQ,OAAO,aAAa,QAAQ,IAIlC;SAAA,UAAU,aAAa,QAAQ,MAAM,6BAA6B,GAElE,GAAG,OACD,uBAAuB,QAAQ,GAAG,GAAG,QAAQ,KAAK,kCAClD,EAAC,QAAO,CACV;IAAA;IAIJ,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,IACE,cAAc,iBACd,UAAU,YAAY,QAAQ,MAAM,wBAAwB,GAE5D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,yBAAyB,WAAW,KAC1E,EAAC,QAAO,CACV;KAEF,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,OACF;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,MAAM,WAAW,aAAa,cAAc;OAC5C,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;QACjB,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,MAAM,WAAW,aAAa,cAAc;EAC5C,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,CAAC,CAC5D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}
1	+ {"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport type {ReplicaConstraints} from '../../litestream/commands.ts';\nimport {\n getSubscriptionStateAndContext,\n type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n restoreReplica,\n type ConstrainingPurgeLock,\n type InitializeResult,\n type InitializeTimings,\n type RestoreOptions,\n type ResumableBackup,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n initialSync,\n type InitialSyncOptions,\n type ReplicaOptions,\n type ServerContext,\n} from './initial-sync.ts';\nimport {endSession} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n claimSlotForResumption,\n createReplicaAndSlot,\n type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n dropShard,\n getActiveReplicas,\n getReplicaAtVersion,\n getRestoreCandidates,\n initRestoreReplica,\n internalPublicationPrefix,\n type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n // Create a new slot for the replica rather than taking over\n // an existing one (i.e. high-availability mode).\n slotPerReplica: boolean \| undefined;\n\n inactiveReplicaGracePeriodMs: number;\n\n /*\n Whether this replication-manager maintains the PG change log, i.e. acks\n * the replication slot only after transactions are stored in it. Recorded\n * with the replica (see `pgChangeLog` in `ReplicaState`) when its slot is\n * taken over.\n /\n pgChangeLog: boolean;\n};\n\n/\n Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n /\nexport async function initializePostgresChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n syncOptions: InitialSyncOptions,\n context: ServerContext,\n lagReportIntervalMs = 0,\n restoreOptions: RestoreOptions = {},\n {\n epoch,\n slotPerReplica,\n backupV5,\n inactiveReplicaGracePeriodMs,\n pgChangeLog,\n }: InitializeOptions = {\n epoch: 0,\n slotPerReplica: false,\n backupV5: true,\n inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n pgChangeLog: false,\n },\n streamInboundTimeoutMs?: number \| undefined,\n): Promise<InitializeResult> {\n const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n try {\n await ensureShardSchema(\n lc,\n db,\n shard,\n syncOptions.installPartialIndexTriggers,\n );\n\n const {\n replica: restoredReplica,\n purgeLock,\n pgChangeLogBehindSlot,\n resumableBackup,\n slotWatermark,\n timings,\n } = slotPerReplica\n ? await forkOrResumeReplica(\n lc,\n db,\n shard,\n epoch,\n syncOptions.replicationSlotFailover ?? false,\n replicaDbFile,\n restoreOptions,\n inactiveReplicaGracePeriodMs,\n )\n : await selectAndRestoreReplica(\n lc,\n db,\n shard,\n replicaDbFile,\n restoreOptions,\n );\n\n let initialSyncedReplica: ReplicaState \| undefined;\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n async (log, tx) => {\n // Falling back to initial sync abandons the restore, so release the\n // resources acquired for it, e.g. a replication slot claimed (or\n // created) for a resumed (or forked) replica, which would otherwise\n // remain active indefinitely.\n await restoreOptions.cleanup?.release();\n // The purge lock on the change-db must be released before performing\n // initial sync; if the change-db and upstream are the same db, a lock-holding\n // transaction will prevent a replication slot from being created. This awkward\n // dependency can go away with RMv2. (The purge lock is also released by\n // the cleanup when it is registered with it; releasing it is idempotent.)\n await purgeLock?.release();\n initialSyncedReplica = await initialSync(\n log,\n shard,\n tx,\n upstreamURI,\n syncOptions,\n context,\n {epoch, backupV5, pgChangeLog},\n restoreOptions.cleanup,\n );\n },\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionStateAndContext(\n new StatementRunner(replica),\n );\n replica.close();\n\n // Check that upstream is properly setup, and throw an AutoReset to re-run\n // initial sync if not.\n const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n lc,\n db,\n shard,\n subscriptionState,\n (initialSyncedReplica ?? restoredReplica)?.id,\n );\n\n // With litestream v5, every replication-manager backs up to its own\n // backup lineage.\n const newBackupLineage = backupV5 && !resumableBackup;\n const backupPath = initialSyncedReplica\n ? // If initial sync was performed, use that initial backupPath.\n initialSyncedReplica.backupPath\n : // Otherwise, use the lineage prepared by the restore, or a new, unique\n // path when backing up with litestream v5. This will be recorded in the\n // replicas table by the PostgresChangeSource.\n backupV5\n ? (resumableBackup?.backupPath ?? String(Date.now()))\n : (restoredReplica?.backupPath ?? null);\n\n const changeSource = new PostgresChangeSource(\n lc,\n upstreamURI,\n shard,\n upstreamReplica,\n pgVersion,\n {backupPath, backupV5, pgChangeLog},\n context,\n lagReportIntervalMs,\n syncOptions.textCopy,\n streamInboundTimeoutMs,\n minStartWatermark(\n subscriptionState.watermark,\n initialSyncedReplica !== undefined,\n slotWatermark,\n purgeLock,\n ),\n );\n\n const destinationBackupURL =\n backupPath && restoreOptions.litestream?.backupURL\n ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n restoreOptions.litestream?.backupURL;\n\n return {\n subscriptionState,\n changeSource,\n destinationBackupURL,\n // The replica this change stream belongs to. It is part of the identity\n // the SQLite change log records, because a generation (i.e.\n // `replicaVersion`) is shared by every sibling of a forked replica and so\n // cannot distinguish two siblings' logs.\n replicaID: upstreamReplica.id,\n waitForBackupBeforeServing:\n // Wait for the first backup if there was an initial sync,\n initialSyncedReplica !== undefined \|\|\n // or if the destination differs from where it was restored\n // (i.e. backupV5).\n backupPath !== (restoredReplica?.backupPath ?? null),\n pgChangeLogBehindSlot,\n newBackupLineage,\n startTimings: initialSyncedReplica\n ? {kind: 'initial_sync'}\n : (timings ?? {kind: slotPerReplica ? 'fork' : 'rmv1'}),\n };\n } finally {\n await db.end();\n }\n}\n\ntype RestoredReplica = {\n replica: ReplicaState \| undefined;\n purgeLock: ConstrainingPurgeLock \| null;\n pgChangeLogBehindSlot?: boolean \| undefined;\n resumableBackup?: ResumableBackup \| undefined;\n /\n The position (as a watermark) of the replication slot from which\n * replication resumes, when it was created (for a fork) or claimed (for a\n * resumption), i.e. while it is held by this process. Absent for a shared\n * slot (RMv1), whose position is advanced by its current owner until it is\n * taken over, and for a resumed slot that was acked only once transactions\n * were stored in the PG change-log (RMv1.5), whose position may be past the\n * change-log's head for acked keepalives.\n /\n slotWatermark?: string \| undefined;\n timings?: InitializeTimings \| undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n// intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n replicaFile: string,\n {litestream, acquirePurgeLock}: RestoreOptions,\n): Promise<RestoredReplica> {\n // The purge lock constrains the generation of the replica to restore. With\n // a shared replication slot, the change-log is resumed from its head\n // without being checked against the slot's position: the slot is still\n // being acked by the previous replication-manager, on an idle upstream past\n // the change-log's head (see PgChangeLogPurgeLocker).\n const lock = await acquirePurgeLock?.();\n assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n const purgeLock = lock ?? null;\n const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n // filter to the generation specified by the purge lock, if present\n ({generation}) => generation === (purgeLock?.replicaVersion ?? generation),\n );\n if (replicas.length === 0) {\n lc.info?.(`no suitable replicas to restore from`, {replicas});\n return {replica: undefined, purgeLock};\n }\n const [replica] = replicas;\n\n if (litestream?.backupURL) {\n const constraints = restoreConstraints(purgeLock, replica);\n const {backupURL: backupBaseURL} = litestream;\n const {slot, backupPath, confirmedFlushLsn} = replica;\n const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n lc.info?.(\n `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n {replicas},\n );\n const restoreStart = performance.now();\n const resumableBackup = await restoreReplica(\n lc,\n {...litestream, backupURL}, // includes the replica's backup sub-path\n replicaFile,\n constraints,\n litestream.forkBackup\n ? {type: 'fork', baseURL: backupBaseURL}\n : undefined,\n );\n const restoreMs = performance.now() - restoreStart;\n return {\n replica,\n purgeLock,\n resumableBackup,\n timings: {kind: 'rmv1', restoreMs},\n };\n }\n return {replica, purgeLock, timings: {kind: 'rmv1'}};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n epoch: number,\n slotFailover: boolean,\n replicaFile: string,\n {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n gracePeriodMs: number,\n): Promise<RestoredReplica> {\n const forkWaitStart = performance.now();\n const result = await getSourceAndDestinationReplicas(\n lc,\n sql,\n shard,\n epoch,\n slotFailover,\n gracePeriodMs,\n // The PG change-log is checked against the slot's position (see below)\n // only when this replication-manager maintains it.\n acquirePurgeLock !== undefined,\n );\n if (!result) {\n // can't restore, must initial-sync\n return {replica: undefined, purgeLock: null};\n }\n const {restoreFrom, replicateTo, slotLSN, slotSession, slotCreateMs} = result;\n const timings: InitializeTimings = {\n kind: restoreFrom.id === replicateTo.id ? 'resume' : 'fork',\n slotCreateMs,\n forkWaitMs: performance.now() - forkWaitStart - (slotCreateMs ?? 0),\n };\n // A resumed slot (i.e. of the same replica) that was acked only once\n // transactions were stored in the PG change-log (RMv1.5) cannot have been\n // acked past a transaction missing from the change-log. Its position past\n // the change-log's head only reflects acked keepalives, so it is not\n // compared with the head (see below).\n const resumedAfterPgChangeLog =\n restoreFrom.id === replicateTo.id && restoreFrom.pgChangeLog;\n const slotWatermark = resumedAfterPgChangeLog\n ? undefined\n : toStateVersionString(slotLSN);\n // The slot is held active by a session in this process until it is taken\n // over by the PostgresChangeSource. If initialization fails before then,\n // the session must be released, or the slot remains active indefinitely,\n // preventing the replica from being resumed (or forked) by a retry.\n cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () =>\n slotSession.destroy(),\n );\n\n // Acquired only after the replication slot has been created (or claimed),\n // and only if the change-log can be resumed from the slot's position.\n //\n // For a fork, this is the case whenever the change-log is maintained (i.e.\n // by a replication-manager that acks the slot only once a transaction has\n // been stored in the change-log): the fork waits for the source replica's\n // slot to pass the position after the creation of the new replica, which is\n // a transaction (in the publication) after the new slot's position (see\n // `waitForStoredTxPastSlot` in getSourceAndDestinationReplicas()).\n //\n // For a resumed slot, the slot can also have been acked past the head of a\n // maintained change-log, for keepalives (e.g. for changes outside of the\n // publication, or on an idle upstream). If the slot was acked by a\n // replication-manager that maintains the change-log, it is therefore not\n // checked against the slot's position, which would otherwise re-initialize\n // the change-log unnecessarily, resetting subscribers behind the replica.\n // Otherwise (e.g. RMv2, or unknown), it is: the change-log is then\n // re-initialized from the replica if it does not contain the replica's\n // changes.\n const lock = acquirePurgeLock ? await acquirePurgeLock(slotWatermark) : null;\n const pgChangeLogBehindSlot = lock === 'behind-slot';\n const purgeLock = lock === 'behind-slot' ? null : lock;\n // Absent a purge lock, still constrain the restore to the source's\n // generation, so that a local replica left behind by a different\n // generation (e.g. a failed initial sync) is not mistaken for it.\n const constraints = restoreConstraints(purgeLock, restoreFrom);\n\n if (litestream?.backupURL) {\n const {backupURL: backupBaseURL} = litestream;\n const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n lc.info?.(\n `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n {restoreFrom, replicateTo},\n );\n const restoreStart = performance.now();\n const resumableBackup = await restoreReplica(\n lc,\n {...litestream, backupURL}, // includes the replica's backup sub-path\n replicaFile,\n constraints,\n litestream.forkBackup\n ? {type: 'fork', baseURL: backupBaseURL}\n : undefined,\n );\n timings.restoreMs = performance.now() - restoreStart;\n return {\n replica: replicateTo,\n purgeLock,\n pgChangeLogBehindSlot,\n resumableBackup,\n slotWatermark,\n timings,\n };\n }\n return {\n replica: replicateTo,\n purgeLock,\n pgChangeLogBehindSlot,\n slotWatermark,\n timings,\n };\n}\n\n/\n The watermark below which the replication stream cannot (correctly) be\n * started, i.e. from which it would skip transactions that the change log\n * does not contain. It reflects the order in which each mode stores a\n * transaction and then acks it to the slot, such that a correctly initialized\n * stream always starts at or past it:\n \n * After an initial sync, the replica's watermark, which is the position\n * at which the new slot was created, and the change log is (re)initialized\n * at it.\n * * With a slot per replica (RMv1.5 and RMv2), the minimum of the restored\n * replica's watermark and the slot's position. Transactions are only acked\n * once backed up (so a skipped interval ends at or before the replica), and\n * if the change log is behind the slot it is re-initialized from the\n * replica. Note that the slot can be past the replica (acked for keepalives,\n * with nothing skipped), and the replica past the change log (forwarded\n * before being stored, and streamed again from the slot), but the stream\n * never starts below both. (With RMv1.5's purge lock, the change log's head\n * is at or past the slot, so it does not lower the minimum.)\n * * With a shared slot (RMv1), or a resumed slot that was acked only once\n * transactions were stored in the change log (RMv1.5), the change log's\n * head when it was purge locked, as the stream resumes from the head, which\n * only advances. The slot is not a reference: a shared slot is acked by its\n * owner until it is taken over, and a resumed RMv1.5 slot can be past the\n * head for acked keepalives. Without a purge lock (i.e. an empty change\n * log), there is no floor.\n /\n// Exported for testing.\nexport function minStartWatermark(\n replicaWatermark: string,\n initialSynced: boolean,\n slotWatermark: string \| undefined,\n purgeLock: ConstrainingPurgeLock \| null,\n): string \| undefined {\n if (initialSynced) {\n return replicaWatermark;\n }\n if (slotWatermark !== undefined) {\n return replicaWatermark < slotWatermark ? replicaWatermark : slotWatermark;\n }\n return purgeLock?.headWatermark;\n}\n\nfunction restoreConstraints(\n purgeLock: ConstrainingPurgeLock \| null,\n {generation}: ReplicaState,\n): ReplicaConstraints {\n return purgeLock ?? {replicaVersion: generation, minWatermark: ''};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 500;\n// Polling is frequent so that a fork proceeds as soon as the source replica\n// passes the fork LSN; logging is throttled by time rather than per poll.\nconst PROGRESS_LOG_INTERVAL_MS = 5_000;\nconst SLOW_PROGRESS_LOG_INTERVAL_MS = 60_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n epoch: number,\n slotFailover: boolean,\n gracePeriodMs: number,\n /\n Whether a fork waits for the source replica's slot to pass a transaction\n * after the destination slot's position (rather than just the position),\n * which is necessary for the PG change-log to be resumable from the\n * destination slot (see forkOrResumeReplica()).\n /\n waitForStoredTxPastSlot: boolean,\n pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n \| {\n restoreFrom: ReplicaState;\n replicateTo: ReplicaState;\n /* The position of the slot from which replication resumes. /\n slotLSN: LSN;\n /\n The session holding the destination slot active until it is taken\n * over by the PostgresChangeSource. Destroying it releases the slot.\n /\n slotSession: Readable;\n /\n The time spent creating the destination replica and slot, if one was\n * created (even if it was then abandoned for a resumption).\n /\n slotCreateMs?: number \| undefined;\n }\n \| undefined\n> {\n const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n let destination: ReplicationSlotResult<void> \| undefined;\n let slotCreateMs: number \| undefined;\n // The position that the source replica's slot must reach to be forked.\n let forkLSN: LSN \| undefined;\n const lastLogged = new Map<string, number>();\n const shouldLog = (key: string, intervalMs: number) => {\n const now = Date.now();\n const last = lastLogged.get(key);\n if (last !== undefined && now - last < intervalMs) {\n return false;\n }\n lastLogged.set(key, now);\n return true;\n };\n\n try {\n for (let i = 0; ; i++) {\n if (i > 0) {\n await sleep(pollIntervalMs);\n }\n const replicas = await getRestoreCandidates(sql, shard, epoch);\n if (shouldLog('replicas', PROGRESS_LOG_INTERVAL_MS)) {\n lc.info?.(`current replicas at epoch ${epoch}`, {replicas});\n }\n if (replicas.length === 0) {\n lc.info?.(`no suitable replicas to restore from`, {replicas});\n await endSession(destination?.initialSession);\n return undefined;\n }\n for (const replica of replicas) {\n // Track inactivity to resume replicas after a grace period.\n if (replica.active) {\n inactiveSince.delete(replica.id);\n } else if (!inactiveSince.has(replica.id)) {\n inactiveSince.set(replica.id, Date.now());\n }\n\n // Note: Only `active` InitialSync replicas are returned from\n // getRestoreCandidates().\n if (replica.stage === InitialSync) {\n // Log periodically; initial-sync can be long\n if (\n shouldLog(\n `initial-sync:${replica.id}`,\n SLOW_PROGRESS_LOG_INTERVAL_MS,\n )\n ) {\n lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n }\n break;\n }\n\n if (replica.stage === Replicate) {\n const {active, confirmedFlushLsn} = replica;\n if (active) {\n // Create a replication slot to fork the active replica.\n if (!destination) {\n const slotCreateStart = performance.now();\n destination = await createReplicaAndSlot(\n lc,\n sql,\n 'fork-replica-session',\n shard,\n epoch,\n Date.now().toString(), // replicaID\n slotFailover,\n {\n backupPath: null, // set only after the backup has been confirmed\n backupV5: true, // RMv2 requires backupV5\n // The slot is not acked until it is taken over, which sets\n // this. Until then, it is unknown.\n pgChangeLog: false,\n },\n () => promiseVoid,\n ReplicaStage.Restore,\n );\n slotCreateMs = performance.now() - slotCreateStart;\n }\n // The source replica acks its slot only once its changes are\n // backed up (and, if it maintains the PG change-log, stored), so\n // its slot reaching the destination slot's position means that\n // the destination can be restored from its backup.\n //\n // If the PG change-log is to be resumed from the destination slot,\n // the fork waits for the source to pass the position after the\n // creation of the destination replica instead. Its row is\n // inserted in a transaction (in the publication) after the\n // destination slot's position, so this ensures that a source\n // replica that maintains the PG change-log has stored a\n // transaction past the destination slot's position, rather than\n // only acked keepalives past it.\n //\n // Otherwise, the ack of that transaction (at its commit LSN, which\n // precedes the position after it) suffices, which the fork request\n // (see fork-request.ts) prompts the source to make promptly.\n forkLSN ??= waitForStoredTxPastSlot\n ? await currentWalLSN(sql)\n : destination.slot.consistent_point;\n if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n if (shouldLog(`fork:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)) {\n lc.info?.(\n `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n {replica},\n );\n }\n break;\n }\n }\n\n // If the replica is past the destination LSN, fork it, regardless of\n // whether its slot is still active. Forking is preferable to resuming,\n // as resuming removes the replica from being a candidate for\n // subsequent forks, and carries the risk of stealing the slot from a\n // task attempting to reconnect.\n if (\n destination &&\n forkLSN &&\n toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n ) {\n lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n replica,\n });\n const replicateTo = must(\n await initRestoreReplica(sql, shard, {\n sourceID: replica.id,\n destID: destination.replica.id,\n }),\n `replica ${destination.replica.id} was deleted`,\n );\n return {\n restoreFrom: replica,\n replicateTo,\n slotLSN: destination.slot.consistent_point,\n slotSession: destination.initialSession,\n slotCreateMs,\n };\n }\n }\n\n if (\n replica.stage === ReplicaStage.Restore &&\n replica.active &&\n replica.id !== destination?.replica.id\n ) {\n // Another task is restoring (e.g. resuming) this replica.\n if (\n shouldLog(`restoring:${replica.id}`, SLOW_PROGRESS_LOG_INTERVAL_MS)\n ) {\n lc.info?.(\n `waiting for replica ${replica.id}@${replica.slot} being restored by another task`,\n {replica},\n );\n }\n }\n\n if (replica.backupPath && !replica.active) {\n // An inactive replica that has a backupPath may be orphaned, or it\n // may be an active task that was temporarily disconnected and\n // attempting to reestablish a session.\n //\n // Resume the replica after a grace period to avoid stealing the slot\n // from an active task that was temporarily disconnected. Resumption\n // is a last resort only taken in the absence of alternatives.\n const now = Date.now();\n const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n if (\n inactiveMs >= gracePeriodMs \|\|\n shouldLog(`inactive:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)\n ) {\n lc.info?.(\n `replica ${replica.id}@${replica.slot} has been inactive for ${inactiveMs}ms`,\n {replica},\n );\n }\n if (inactiveMs >= gracePeriodMs) {\n const reserved = await claimSlotForResumption(\n lc,\n sql,\n shard,\n 'resume-replica',\n replica,\n );\n if (!reserved) {\n lc.warn?.(\n `unable to resume replica ${replica.id}@${replica.slot}`,\n );\n } else {\n lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n // If a new replication slot was created in anticipation of forking,\n // cancel it\n await endSession(destination?.initialSession);\n return {\n restoreFrom: replica,\n replicateTo: reserved.replica,\n // The slot is inactive, so its position is fixed.\n slotLSN: replica.confirmedFlushLsn,\n slotSession: reserved.reservation,\n slotCreateMs,\n };\n }\n }\n }\n }\n }\n } catch (e) {\n await endSession(destination?.initialSession);\n throw e;\n }\n}\n\nasync function checkAndUpdateUpstream(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n {\n replicaVersion,\n publications: subscribed,\n initialSyncContext,\n }: SubscriptionStateAndContext,\n replicaID: string \| undefined,\n) {\n const upstreamReplica = await getReplicaAtVersion(\n lc,\n sql,\n shard,\n replicaVersion,\n replicaID,\n initialSyncContext,\n );\n if (!upstreamReplica) {\n throw new AutoResetSignal(\n `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n );\n }\n\n // Verify that the publications match what is being replicated.\n const requested = shard.publications.toSorted();\n const replicated = upstreamReplica.publications\n .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n .sort();\n if (!deepEqual(requested, replicated)) {\n lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n throw new AutoResetSignal(\n `Requested publications [${requested}] do not match configured ` +\n `publications: [${replicated}]`,\n );\n }\n\n // Sanity check: The subscription state on the replica should have the\n // same publications. This should be guaranteed by the equivalence of the\n // replicaVersion, but it doesn't hurt to verify.\n if (!deepEqual(upstreamReplica.publications, subscribed)) {\n throw new AutoResetSignal(\n `Upstream publications [${upstreamReplica.publications}] do not ` +\n `match subscribed publications [${subscribed}]`,\n );\n }\n\n // Verify that the publications exist.\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n `.values();\n if (exists.length !== subscribed.length) {\n throw new AutoResetSignal(\n `Upstream publications [${exists.flat()}] do not contain ` +\n `all subscribed publications [${subscribed}]`,\n );\n }\n\n const {slot} = upstreamReplica;\n const result = await sql<{restartLSN: LSN \| null; walStatus: string \| null}[]>\n /sql/ `\n SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n WHERE slot_name = ${slot}`;\n if (result.length === 0) {\n throw new AutoResetSignal(`replication slot ${slot} is missing`);\n }\n const [{restartLSN, walStatus}] = result;\n if (restartLSN === null \|\| walStatus === 'lost') {\n throw new AutoResetSignal(\n `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n );\n }\n const [{pgVersion}] = await sql<{pgVersion: number}[]> /sql*/ `\n SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AA0EA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,8BACA,gBACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;CAC9B,aAAa;AACf,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,uBACA,iBACA,eACA,YACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,MAAM,eAAe,SAAS,QAAQ;GAMtC,MAAM,WAAW,QAAQ;GACzB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;IAAU;GAAW,GAC7B,eAAe,OACjB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,gBAAA,EAAkB,EAC7C;EAIA,MAAM,mBAAmB,YAAY,CAAC;EACtC,MAAM,aAAa,uBAEf,qBAAqB,aAIrB,WACG,iBAAiB,cAAc,OAAO,KAAK,IAAI,CAAC,IAChD,iBAAiB,cAAc;EA2BtC,OAAO;GACL;GACA,cAAA,IA3BuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;IAAU;GAAW,GAClC,SACA,qBACA,YAAY,UACZ,wBACA,kBACE,kBAAkB,WAClB,yBAAyB,KAAA,GACzB,eACA,SACF,CAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,CAAC,CAAC,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;GACA;GACA,cAAc,uBACV,EAAC,MAAM,eAAc,IACpB,WAAW,EAAC,MAAM,iBAAiB,SAAS,OAAM;EACzD;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAsBA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,oBACa;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,EAAA,CAAG,QAExD,EAAC,iBAAgB,gBAAgB,WAAW,kBAAkB,WACjE;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,cAAc,mBAAmB,WAAW,OAAO;EACzD,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,CAAC,CAAC,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EACA,MAAM,eAAe,YAAY,IAAI;EAWrC,OAAO;GACL;GACA;GACA,iBAAA,MAb4B,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,CACN;GAME,SAAS;IAAC,MAAM;IAAQ,WALR,YAAY,IAAI,IAAI;GAKH;EACnC;CACF;CACA,OAAO;EAAC;EAAS;EAAW,SAAS,EAAC,MAAM,OAAM;CAAC;AACrD;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,kBAAkB,WAC/B,eAC0B;CAC1B,MAAM,gBAAgB,YAAY,IAAI;CACtC,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,eAGA,qBAAqB,KAAA,CACvB;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,SAAS,aAAa,iBAAgB;CACvE,MAAM,UAA6B;EACjC,MAAM,YAAY,OAAO,YAAY,KAAK,WAAW;EACrD;EACA,YAAY,YAAY,IAAI,IAAI,iBAAiB,gBAAgB;CACnE;CAQA,MAAM,gBADJ,YAAY,OAAO,YAAY,MAAM,YAAY,cAE/C,KAAA,IACA,qBAAqB,OAAO;CAKhC,SAAS,UAAU,oBAAoB,YAAY,cACjD,YAAY,QAAQ,CACtB;CAqBA,MAAM,OAAO,mBAAmB,MAAM,iBAAiB,aAAa,IAAI;CACxE,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAIlD,MAAM,cAAc,mBAAmB,WAAW,WAAW;CAE7D,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,CAAC,CAAC,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EACA,MAAM,eAAe,YAAY,IAAI;EACrC,MAAM,kBAAkB,MAAM,eAC5B,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,aACA,WAAW,aACP;GAAC,MAAM;GAAQ,SAAS;EAAa,IACrC,KAAA,CACN;EACA,QAAQ,YAAY,YAAY,IAAI,IAAI;EACxC,OAAO;GACL,SAAS;GACT;GACA;GACA;GACA;GACA;EACF;CACF;CACA,OAAO;EACL,SAAS;EACT;EACA;EACA;EACA;CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,kBACd,kBACA,eACA,eACA,WACoB;CACpB,IAAI,eACF,OAAO;CAET,IAAI,kBAAkB,KAAA,GACpB,OAAO,mBAAmB,gBAAgB,mBAAmB;CAE/D,OAAO,WAAW;AACpB;AAEA,SAAS,mBACP,WACA,EAAC,cACmB;CACpB,OAAO,aAAa;EAAC,gBAAgB;EAAY,cAAc;CAAE;AACnE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AAGjC,IAAM,2BAA2B;AACjC,IAAM,gCAAgC;AACtC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eAOA,yBACA,iBAAiB,0BAmBjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CACJ,IAAI;CAEJ,IAAI;CACJ,MAAM,6BAAa,IAAI,IAAoB;CAC3C,MAAM,aAAa,KAAa,eAAuB;EACrD,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,OAAO,WAAW,IAAI,GAAG;EAC/B,IAAI,SAAS,KAAA,KAAa,MAAM,OAAO,YACrC,OAAO;EAET,WAAW,IAAI,KAAK,GAAG;EACvB,OAAO;CACT;CAEA,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,KAAK,OAAO,KAAK;GAC7D,IAAI,UAAU,YAAY,wBAAwB,GAChD,GAAG,OAAO,6BAA6B,SAAS,EAAC,SAAQ,CAAC;GAE5D,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,MAAM,WAAW,aAAa,cAAc;IAC5C;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IACE,UACE,gBAAgB,QAAQ,MACxB,6BACF,GAEA,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,IAAI,CAAC,aAAa;OAChB,MAAM,kBAAkB,YAAY,IAAI;OACxC,cAAc,MAAM,qBAClB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,CAAC,CAAC,SAAS,GACpB,cACA;QACE,YAAY;QACZ,UAAU;QAGV,aAAa;OACf,SACM,aACN,CACF;OACA,eAAe,YAAY,IAAI,IAAI;MACrC;MAkBA,YAAY,0BACR,MAAM,cAAc,GAAG,IACvB,YAAY,KAAK;MACrB,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,IAAI,UAAU,QAAQ,QAAQ,MAAM,wBAAwB,GAC1D,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OAEF;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;OAC1B,aAAa,YAAY;OACzB;MACF;KACF;IACF;IAEA,IACE,QAAQ,UAAU,KAClB,QAAQ,UACR,QAAQ,OAAO,aAAa,QAAQ,IAIlC;SAAA,UAAU,aAAa,QAAQ,MAAM,6BAA6B,GAElE,GAAG,OACD,uBAAuB,QAAQ,GAAG,GAAG,QAAQ,KAAK,kCAClD,EAAC,QAAO,CACV;IAAA;IAIJ,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,IACE,cAAc,iBACd,UAAU,YAAY,QAAQ,MAAM,wBAAwB,GAE5D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,yBAAyB,WAAW,KAC1E,EAAC,QAAO,CACV;KAEF,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,OACF;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,MAAM,WAAW,aAAa,cAAc;OAC5C,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;QACjB,aAAa,SAAS;QACtB;OACF;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,MAAM,WAAW,aAAa,cAAc;EAC5C,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,CAAC,CAC5D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}

package/out/zero-cache/src/services/change-streamer/server-start-metrics.js CHANGED Viewed

@@ -26,7 +26,7 @@ var ServerStartMetrics = class {
 		this.#ready = true;
 		const kind = this.#kind ?? "unknown";
 		getOrCreateGauge("replication", "server_start", {
-			description: "The duration of each phase of the replication-manager start (fork_wait, restore, catch_up, backup_confirm, and the total from process start to readiness), by the kind of initialization (fork, resume, rmv1, initial_sync).",
+			description: "The duration of each phase of the replication-manager start (slot_create, fork_wait, restore, catch_up, backup_confirm, and the total from process start to readiness), by the kind of initialization (fork, resume, rmv1, initial_sync).",
 			unit: "s"
 		}).addCallback((o) => {
 			for (const [phase, ms] of this.#phasesMs) o.observe(ms / 1e3, {

package/out/zero-cache/src/services/change-streamer/server-start-metrics.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"server-start-metrics.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/server-start-metrics.ts"],"sourcesContent":["import {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {InitializeKind} from '../change-source/common/replica-restore.ts';\n\n/*\n The phases of a replication-manager's start:\n * - `fork_wait`: selecting the replica to restore from (and ~~creating~~ or\n * ~~claiming its~~ slot), including the wait for a forked replica to pass the\n * new slot's position\n * - `restore`: restoring the replica from its backup\n * - `catch_up`: from initialization to the backup-replicator being caught up\n * (i.e. its replication lag readiness)\n * - `backup_confirm`: waiting for the first backup to cover the replica\n * - `total`: from the start of the (change-streamer) process to readiness\n \n The phases are not exhaustive (e.g. connecting to and preparing the\n * upstream), and `catch_up` and `backup_confirm` overlap.\n /\nexport type ServerStartPhase =\n \| 'fork_wait'\n \| 'restore'\n \| 'catch_up'\n \| 'backup_confirm'\n \| 'total';\n\n/\n Records where a replication-manager's start goes, reported once it is\n * ready by the `zero.replication.server_start` gauge, in seconds, labeled\n * by `phase` and `kind` (see {@link InitializeKind}). This is for diagnosis,\n * e.g. of why a start is slow, as the process only starts once.\n /\nexport class ServerStartMetrics {\n readonly #phasesMs = new Map<ServerStartPhase, number>();\n #kind: InitializeKind \| undefined;\n #ready = false;\n\n setKind(kind: InitializeKind \| undefined) {\n this.#kind = kind;\n }\n\n record(phase: ServerStartPhase, ms: number \| undefined) {\n if (ms !== undefined && !this.#ready) {\n this.#phasesMs.set(phase, ms);\n }\n }\n\n /\n Records the `total` (by default, the time since the process started) and\n * starts reporting the gauge. Phases recorded afterwards are ignored.\n */\n ready(totalMs = performance.now()) {\n if (this.#ready) {\n return;\n }\n this.record('total', totalMs);\n this.#ready = true;\n const kind = this.#kind ?? 'unknown';\n getOrCreateGauge('replication', 'server_start', {\n description:\n 'The duration of each phase of the replication-manager start ' +\n '(fork_wait, restore, catch_up, backup_confirm, and the total ' +\n 'from process start to readiness), by the kind of initialization ' +\n '(fork, resume, rmv1, initial_sync).',\n unit: 's',\n }).addCallback(o => {\n for (const [phase, ms] of this.#phasesMs) {\n o.observe(ms / 1000, {phase, kind});\n }\n });\n }\n}\n"],"mappings":";;;;;;;;~~AA8BA~~,IAAa,qBAAb,MAAgC;CAC9B,4BAAqB,IAAI,IAA8B;CACvD;CACA,SAAS;CAET,QAAQ,MAAkC;EACxC,KAAK,QAAQ;CACf;CAEA,OAAO,OAAyB,IAAwB;EACtD,IAAI,OAAO,KAAA,KAAa,CAAC,KAAK,QAC5B,KAAK,UAAU,IAAI,OAAO,EAAE;CAEhC;;;;;CAMA,MAAM,UAAU,YAAY,IAAI,GAAG;EACjC,IAAI,KAAK,QACP;EAEF,KAAK,OAAO,SAAS,OAAO;EAC5B,KAAK,SAAS;EACd,MAAM,OAAO,KAAK,SAAS;EAC3B,iBAAiB,eAAe,gBAAgB;GAC9C,aACE;GAIF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK;GAClB,KAAK,MAAM,CAAC,OAAO,OAAO,KAAK,WAC7B,EAAE,QAAQ,KAAK,KAAM;IAAC;IAAO;GAAI,CAAC;EAEtC,CAAC;CACH;AACF"}
1	+ {"version":3,"file":"server-start-metrics.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/server-start-metrics.ts"],"sourcesContent":["import {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {InitializeKind} from '../change-source/common/replica-restore.ts';\n\n/*\n The phases of a replication-manager's start:\n * - `slot_create`: creating the new replica's slot, including retries while\n * it is blocked by upstream transactions. This is separate from `fork_wait`\n * because the upstream (e.g. a long-running transaction), not zero-cache or\n * its capacity, determines how long it takes.\n * - `fork_wait`: selecting the replica to restore from (and claiming its\n * slot), including the wait for a forked replica to pass the new slot's\n * position, excluding `slot_create`\n * - `restore`: restoring the replica from its backup\n * - `catch_up`: from initialization to the backup-replicator being caught up\n * (i.e. its replication lag readiness)\n * - `backup_confirm`: waiting for the first backup to cover the replica\n * - `total`: from the start of the (change-streamer) process to readiness\n \n The phases are not exhaustive (e.g. connecting to and preparing the\n * upstream), and `catch_up` and `backup_confirm` overlap.\n /\nexport type ServerStartPhase =\n \| 'slot_create'\n \| 'fork_wait'\n \| 'restore'\n \| 'catch_up'\n \| 'backup_confirm'\n \| 'total';\n\n/\n Records where a replication-manager's start goes, reported once it is\n * ready by the `zero.replication.server_start` gauge, in seconds, labeled\n * by `phase` and `kind` (see {@link InitializeKind}). This is for diagnosis,\n * e.g. of why a start is slow, as the process only starts once.\n /\nexport class ServerStartMetrics {\n readonly #phasesMs = new Map<ServerStartPhase, number>();\n #kind: InitializeKind \| undefined;\n #ready = false;\n\n setKind(kind: InitializeKind \| undefined) {\n this.#kind = kind;\n }\n\n record(phase: ServerStartPhase, ms: number \| undefined) {\n if (ms !== undefined && !this.#ready) {\n this.#phasesMs.set(phase, ms);\n }\n }\n\n /\n Records the `total` (by default, the time since the process started) and\n * starts reporting the gauge. Phases recorded afterwards are ignored.\n */\n ready(totalMs = performance.now()) {\n if (this.#ready) {\n return;\n }\n this.record('total', totalMs);\n this.#ready = true;\n const kind = this.#kind ?? 'unknown';\n getOrCreateGauge('replication', 'server_start', {\n description:\n 'The duration of each phase of the replication-manager start ' +\n '(slot_create, fork_wait, restore, catch_up, backup_confirm, and the total ' +\n 'from process start to readiness), by the kind of initialization ' +\n '(fork, resume, rmv1, initial_sync).',\n unit: 's',\n }).addCallback(o => {\n for (const [phase, ms] of this.#phasesMs) {\n o.observe(ms / 1000, {phase, kind});\n }\n });\n }\n}\n"],"mappings":";;;;;;;;AAmCA,IAAa,qBAAb,MAAgC;CAC9B,4BAAqB,IAAI,IAA8B;CACvD;CACA,SAAS;CAET,QAAQ,MAAkC;EACxC,KAAK,QAAQ;CACf;CAEA,OAAO,OAAyB,IAAwB;EACtD,IAAI,OAAO,KAAA,KAAa,CAAC,KAAK,QAC5B,KAAK,UAAU,IAAI,OAAO,EAAE;CAEhC;;;;;CAMA,MAAM,UAAU,YAAY,IAAI,GAAG;EACjC,IAAI,KAAK,QACP;EAEF,KAAK,OAAO,SAAS,OAAO;EAC5B,KAAK,SAAS;EACd,MAAM,OAAO,KAAK,SAAS;EAC3B,iBAAiB,eAAe,gBAAgB;GAC9C,aACE;GAIF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK;GAClB,KAAK,MAAM,CAAC,OAAO,OAAO,KAAK,WAC7B,EAAE,QAAQ,KAAK,KAAM;IAAC;IAAO;GAAI,CAAC;EAEtC,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/mutagen/mutagen.js CHANGED Viewed

@@ -138,8 +138,7 @@ async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, mut
 	}
 	authorizer.reloadPermissions();
 	if (!errorMode) {
-		const { ops } = mutation.args[0];
-		authorizer.validateTableNames(ops);
+		const ops = authorizer.validateOps(mutation.args[0].ops);
 		const normalizedOps = authorizer.normalizeOps(ops);
 		const [canPre, canPost] = await Promise.all([authorizer.canPreMutation(authData, normalizedOps), authorizer.canPostMutation(authData, normalizedOps)]);
 		if (canPre && canPost) for (const op of ops) switch (op.op) {

package/out/zero-cache/src/services/mutagen/mutagen.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"mutagen.js","names":["ErrorKind.MutationRateLimited","MutationType.CRUD","Mode.SERIALIZABLE","ErrorKind.MutationFailed","v.parse","ErrorKind.InvalidPush","ErrorOrigin.ZeroCache"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n isProtocolError,\n ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n type CRUDMutation,\n type DeleteOp,\n type InsertOp,\n type Mutation,\n type UpdateOp,\n type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n primaryKeyValueSchema,\n type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n WriteAuthorizerImpl,\n type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n kind: ErrorKind.MutationFailed \| ErrorKind.MutationRateLimited,\n desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n /*\n Callers must hold a {@link ref} for the duration of a push (i.e. across\n * all of its mutations). The service stops, closing its replica handle,\n * when its ref count drops to zero.\n */\n processMutation(\n mutation: Mutation,\n authData: JWTPayload \| undefined,\n customMutatorsEnabled: boolean,\n ): Promise<MutationError \| undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #upstream: PostgresDB;\n readonly #shard: ShardID;\n readonly #stopped = resolver();\n readonly #replica: Database;\n readonly #writeAuthorizer: WriteAuthorizerImpl;\n readonly #limiter: SlidingWindowLimiter \| undefined;\n #refCount = 0;\n #isStopped = false;\n\n readonly #crudMutations = getOrCreateCounter(\n 'mutation',\n 'crud',\n 'Number of CRUD mutations processed',\n );\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n clientGroupID: string,\n upstream: PostgresDB,\n config: ZeroConfig,\n writeAuthzStorage: DatabaseStorage,\n ) {\n this.id = clientGroupID;\n this.#lc = lc;\n this.#upstream = upstream;\n this.#shard = shard;\n this.#replica = new Database(this.#lc, config.replica.file, {\n fileMustExist: true,\n });\n this.#writeAuthorizer = new WriteAuthorizerImpl(\n this.#lc,\n config,\n this.#replica,\n shard.appID,\n clientGroupID,\n writeAuthzStorage,\n );\n\n if (config.perUserMutationLimit.max !== undefined) {\n this.#limiter = new SlidingWindowLimiter(\n config.perUserMutationLimit.windowMs,\n config.perUserMutationLimit.max,\n );\n }\n }\n\n ref() {\n assert(!this.#isStopped, 'MutagenService is already stopped');\n ++this.#refCount;\n }\n\n unref() {\n assert(!this.#isStopped, 'MutagenService is already stopped');\n --this.#refCount;\n if (this.#refCount <= 0) {\n void this.stop();\n }\n }\n\n hasRefs(): boolean {\n return this.#refCount > 0;\n }\n\n processMutation(\n mutation: Mutation,\n authData: JWTPayload \| undefined,\n customMutatorsEnabled = false,\n ): Promise<MutationError \| undefined> {\n if (this.#limiter?.canDo() === false) {\n return Promise.resolve([\n ErrorKind.MutationRateLimited,\n 'Rate limit exceeded',\n ]);\n }\n this.#crudMutations.add(1, {\n clientGroupID: this.id,\n });\n return processMutation(\n this.#lc,\n authData,\n this.#upstream,\n this.#shard,\n this.id,\n mutation,\n this.#writeAuthorizer,\n undefined,\n customMutatorsEnabled,\n );\n }\n\n run(): Promise<void> {\n return this.#stopped.promise;\n }\n\n stop(): Promise<void> {\n if (this.#isStopped) {\n return this.#stopped.promise;\n }\n this.#isStopped = true;\n try {\n this.#writeAuthorizer.destroy();\n } catch (e) {\n this.#lc.error?.('error destroying write authorizer storage', e);\n } finally {\n // The replica's statistics are maintained by the replicator, so skip\n // the `PRAGMA optimize` that closing a writable handle would otherwise\n // run on every client group churn.\n this.#replica.close({optimize: false});\n }\n this.#stopped.resolve();\n return this.#stopped.promise;\n }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n lc: LogContext,\n authData: JWTPayload \| undefined,\n db: PostgresDB,\n shard: ShardID,\n clientGroupID: string,\n mutation: Mutation,\n writeAuthorizer: WriteAuthorizer,\n onTxStart?: () => void \| Promise<void>, // for testing\n customMutatorsEnabled = false,\n): Promise<MutationError \| undefined> {\n assert(\n mutation.type === MutationType.CRUD,\n 'Only CRUD mutations are supported',\n );\n // log-leak-ignore -- the mutation id, not row data\n lc = lc.withContext('mutationID', mutation.id);\n lc = lc.withContext('processMutation');\n // The mutationID context set above identifies it; the mutation itself\n // carries the row values being written.\n lc.debug?.('Process mutation start');\n\n // Record mutation processing attempt for telemetry (regardless of success/failure)\n recordMutation('crud');\n\n let result: MutationError \| undefined;\n\n const start = Date.now();\n try {\n // Mutations can fail for a variety of reasons:\n //\n // - application error\n // - network/db error\n // - zero bug\n //\n // For application errors what we want is to re-run the mutation in\n // \"error mode\", which skips the actual mutation and just updates the\n // lastMutationID. Then return the error to the app.\n //\n // However, it's hard to tell the difference between application errors\n // and the other types.\n //\n // A reasonable policy ends up being to just retry every mutation once\n // in error mode. If the error mode mutation succeeds then we assume it\n // was an application error and return the error to the app. Otherwise,\n // we know it was something internal and we log it.\n //\n // This is not 100% correct - there are theoretical cases where we\n // return an internal error to the app that shouldn't have been. But it\n // would have to be a crazy coincidence: we'd have to have a network\n // error on the first attempt that resolves by the second attempt.\n //\n // One might ask why not try/catch just the calls to the mutators and\n // consider those application errors. That is actually what we do in\n // Replicache:\n //\n // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n //\n // We don't do it here because:\n //\n // 1. It's still not perfect. It's hard to isolate SQL errors in\n // mutators due to app developer mistakes from SQL errors due to\n // Zero mistakes.\n // 2. It's not possible to do this with the pg library we're using in\n // Zero anyway: https://github.com/porsager/postgres/issues/455.\n //\n // Personally I think this simple retry policy is nice.\n let errorMode = false;\n for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n try {\n await runTx(\n db,\n async tx => {\n // Simulates a concurrent request for testing. In production this is a noop.\n const done = onTxStart?.();\n try {\n return await processMutationWithTx(\n lc,\n tx,\n authData,\n shard,\n clientGroupID,\n mutation,\n errorMode,\n writeAuthorizer,\n );\n } finally {\n await done;\n }\n },\n {mode: Mode.SERIALIZABLE},\n );\n if (errorMode) {\n lc.debug?.('Ran mutation successfully in error mode');\n }\n break;\n } catch (e) {\n if (e instanceof MutationAlreadyProcessedError) {\n lc.debug?.(e.message);\n // Don't double-count already processed mutations, but they were counted above\n return undefined;\n }\n if (\n isProtocolError(e) &&\n !errorMode &&\n e.kind === ErrorKind.InvalidPush &&\n customMutatorsEnabled &&\n i < 2\n ) {\n // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n // This can create a lot of OOO mutation errors since we do not know when the API server\n // has applied a custom mutation before moving on to process CRUD mutations.\n // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n // They should migrate completely to custom mutators.\n lc.info?.(\n 'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n e,\n );\n await new Promise(resolve => setTimeout(resolve, 100));\n continue;\n }\n if (isProtocolError(e) \|\| errorMode) {\n lc.error?.('Process mutation error', e);\n throw e;\n }\n if (\n e instanceof postgres.PostgresError &&\n e.code === PG_SERIALIZATION_FAILURE\n ) {\n lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n }\n result = [ErrorKind.MutationFailed, String(e)];\n if (errorMode) {\n break;\n }\n lc.error?.('Got error running mutation, re-running in error mode', e);\n errorMode = true;\n i--;\n }\n }\n } finally {\n lc.debug?.('Process mutation complete in', Date.now() - start);\n }\n return result;\n}\n\nexport async function processMutationWithTx(\n lc: LogContext,\n tx: PostgresTransaction,\n authData: JWTPayload \| undefined,\n shard: ShardID,\n clientGroupID: string,\n mutation: CRUDMutation,\n errorMode: boolean,\n authorizer: WriteAuthorizer,\n) {\n const tasks: (() => Promise<unknown>)[] = [];\n\n async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n try {\n return await stmt.execute();\n } finally {\n const q = stmt as unknown as Query;\n // `q.parameters` are the bound row values -- log the parameterized\n // statement and the parameter count, never the values themselves.\n lc.debug?.(`${q.string} (${q.parameters.length} params)`);\n }\n }\n\n authorizer.reloadPermissions();\n\n if (!errorMode) {\n const {ops} = mutation.args[0];\n\n // Validate all table names before processing\n authorizer.validateTableNames(ops);\n\n const normalizedOps = authorizer.normalizeOps(ops);\n const [canPre, canPost] = await Promise.all([\n authorizer.canPreMutation(authData, normalizedOps),\n authorizer.canPostMutation(authData, normalizedOps),\n ]);\n if (canPre && canPost) {\n for (const op of ops) {\n switch (op.op) {\n case 'insert':\n tasks.push(() => execute(getInsertSQL(tx, op)));\n break;\n case 'upsert':\n tasks.push(() => execute(getUpsertSQL(tx, op)));\n break;\n case 'update':\n tasks.push(() => execute(getUpdateSQL(tx, op)));\n break;\n case 'delete':\n tasks.push(() => execute(getDeleteSQL(tx, op)));\n break;\n default:\n unreachable(op);\n }\n }\n }\n }\n\n // Confirm the mutation even though it may have been blocked by the authorizer.\n // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n tasks.unshift(() =>\n checkSchemaVersionAndIncrementLastMutationID(\n tx,\n shard,\n clientGroupID,\n mutation.clientID,\n mutation.id,\n ),\n );\n\n // Note: An error thrown from any Promise aborts the entire transaction.\n await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n tx: postgres.TransactionSql,\n create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n return tx`\n INSERT INTO ${tx(create.tableName)} ${tx(create.value)}\n ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING\n `;\n}\n\nexport function getUpsertSQL(\n tx: postgres.TransactionSql,\n set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const {tableName, primaryKey, value} = set;\n return tx`\n INSERT INTO ${tx(tableName)} ${tx(value)}\n ON CONFLICT (${tx(primaryKey)})\n DO UPDATE SET ${tx(value)}\n `;\n}\n\nfunction getUpdateSQL(\n tx: postgres.TransactionSql,\n update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const table = update.tableName;\n const {primaryKey, value} = update;\n const id: Record<string, PrimaryKeyValue> = {};\n for (const key of primaryKey) {\n id[key] = v.parse(value[key], primaryKeyValueSchema);\n }\n return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n id,\n ).flatMap(([key, value], i) =>\n i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n )}`;\n}\n\nfunction getDeleteSQL(\n tx: postgres.TransactionSql,\n deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const {tableName, primaryKey, value} = deleteOp;\n\n const conditions = [];\n for (const key of primaryKey) {\n if (conditions.length > 0) {\n conditions.push(tx`AND`);\n }\n conditions.push(tx`${tx(key)} = ${value[key]}`);\n }\n\n return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n tx: PostgresTransaction,\n shard: ShardID,\n clientGroupID: string,\n clientID: string,\n receivedMutationID: number,\n) {\n const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n INSERT INTO ${tx(upstreamSchema(shard))}.clients \n as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n VALUES (${clientGroupID}, ${clientID}, ${1})\n ON CONFLICT (\"clientGroupID\", \"clientID\")\n DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n RETURNING \"lastMutationID\"\n `;\n\n // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n if (receivedMutationID < lastMutationID) {\n throw new MutationAlreadyProcessedError(\n clientID,\n receivedMutationID,\n lastMutationID,\n );\n } else if (receivedMutationID > lastMutationID) {\n throw new ProtocolError({\n kind: ErrorKind.InvalidPush,\n message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n origin: ErrorOrigin.ZeroCache,\n });\n }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAK,MAAM;EACX,KAAK,YAAY;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,SAAS,KAAK,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAK,mBAAmB,IAAI,oBAC1B,KAAK,KACL,QACA,KAAK,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAK,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;EACP,IAAI,KAAK,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAK,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAK,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrBA,qBACA,qBACF,CAAC;EAEH,KAAK,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAK,KACL,UACA,KAAK,WACL,KAAK,QACL,KAAK,IACL,UACA,KAAK,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAK,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAK,YACP,OAAO,KAAK,SAAS;EAEvB,KAAK,aAAa;EAClB,IAAI;GACF,KAAK,iBAAiB,QAAQ;EAChC,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,6CAA6C,CAAC;EACjE,UAAU;GAIR,KAAK,SAAS,MAAM,EAAC,UAAU,MAAK,CAAC;EACvC;EACA,KAAK,SAAS,QAAQ;EACtB,OAAO,KAAK,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAASC,MAClB,mCACF;CAEA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CAGrC,GAAG,QAAQ,wBAAwB;CAGnC,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAMC,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAAQ,EAAE,OAAO;IAEpB;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAACC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GAGV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,EAAE,WAAW,OAAO,SAAS;EAC1D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EACd,MAAM,EAAC,QAAO,SAAS,KAAK;EAG5B,WAAW,mBAAmB,GAAG;EAEjC,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAgB,aACd,IACA,QACuC;CACvC,OAAO,EAAE;kBACO,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK,EAAE;mBACxC,GAAG,OAAO,UAAU,EAAE;;AAEzC;AAEA,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAOC,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAMC;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQC;CACV,CAAC;AAEL"}
1	+ {"version":3,"file":"mutagen.js","names":["ErrorKind.MutationRateLimited","MutationType.CRUD","Mode.SERIALIZABLE","ErrorKind.MutationFailed","v.parse","ErrorKind.InvalidPush","ErrorOrigin.ZeroCache"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n isProtocolError,\n ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n type CRUDMutation,\n type DeleteOp,\n type InsertOp,\n type Mutation,\n type UpdateOp,\n type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n primaryKeyValueSchema,\n type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n WriteAuthorizerImpl,\n type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n kind: ErrorKind.MutationFailed \| ErrorKind.MutationRateLimited,\n desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n /*\n Callers must hold a {@link ref} for the duration of a push (i.e. across\n * all of its mutations). The service stops, closing its replica handle,\n * when its ref count drops to zero.\n */\n processMutation(\n mutation: Mutation,\n authData: JWTPayload \| undefined,\n customMutatorsEnabled: boolean,\n ): Promise<MutationError \| undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #upstream: PostgresDB;\n readonly #shard: ShardID;\n readonly #stopped = resolver();\n readonly #replica: Database;\n readonly #writeAuthorizer: WriteAuthorizerImpl;\n readonly #limiter: SlidingWindowLimiter \| undefined;\n #refCount = 0;\n #isStopped = false;\n\n readonly #crudMutations = getOrCreateCounter(\n 'mutation',\n 'crud',\n 'Number of CRUD mutations processed',\n );\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n clientGroupID: string,\n upstream: PostgresDB,\n config: ZeroConfig,\n writeAuthzStorage: DatabaseStorage,\n ) {\n this.id = clientGroupID;\n this.#lc = lc;\n this.#upstream = upstream;\n this.#shard = shard;\n this.#replica = new Database(this.#lc, config.replica.file, {\n fileMustExist: true,\n });\n this.#writeAuthorizer = new WriteAuthorizerImpl(\n this.#lc,\n config,\n this.#replica,\n shard.appID,\n clientGroupID,\n writeAuthzStorage,\n );\n\n if (config.perUserMutationLimit.max !== undefined) {\n this.#limiter = new SlidingWindowLimiter(\n config.perUserMutationLimit.windowMs,\n config.perUserMutationLimit.max,\n );\n }\n }\n\n ref() {\n assert(!this.#isStopped, 'MutagenService is already stopped');\n ++this.#refCount;\n }\n\n unref() {\n assert(!this.#isStopped, 'MutagenService is already stopped');\n --this.#refCount;\n if (this.#refCount <= 0) {\n void this.stop();\n }\n }\n\n hasRefs(): boolean {\n return this.#refCount > 0;\n }\n\n processMutation(\n mutation: Mutation,\n authData: JWTPayload \| undefined,\n customMutatorsEnabled = false,\n ): Promise<MutationError \| undefined> {\n if (this.#limiter?.canDo() === false) {\n return Promise.resolve([\n ErrorKind.MutationRateLimited,\n 'Rate limit exceeded',\n ]);\n }\n this.#crudMutations.add(1, {\n clientGroupID: this.id,\n });\n return processMutation(\n this.#lc,\n authData,\n this.#upstream,\n this.#shard,\n this.id,\n mutation,\n this.#writeAuthorizer,\n undefined,\n customMutatorsEnabled,\n );\n }\n\n run(): Promise<void> {\n return this.#stopped.promise;\n }\n\n stop(): Promise<void> {\n if (this.#isStopped) {\n return this.#stopped.promise;\n }\n this.#isStopped = true;\n try {\n this.#writeAuthorizer.destroy();\n } catch (e) {\n this.#lc.error?.('error destroying write authorizer storage', e);\n } finally {\n // The replica's statistics are maintained by the replicator, so skip\n // the `PRAGMA optimize` that closing a writable handle would otherwise\n // run on every client group churn.\n this.#replica.close({optimize: false});\n }\n this.#stopped.resolve();\n return this.#stopped.promise;\n }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n lc: LogContext,\n authData: JWTPayload \| undefined,\n db: PostgresDB,\n shard: ShardID,\n clientGroupID: string,\n mutation: Mutation,\n writeAuthorizer: WriteAuthorizer,\n onTxStart?: () => void \| Promise<void>, // for testing\n customMutatorsEnabled = false,\n): Promise<MutationError \| undefined> {\n assert(\n mutation.type === MutationType.CRUD,\n 'Only CRUD mutations are supported',\n );\n // log-leak-ignore -- the mutation id, not row data\n lc = lc.withContext('mutationID', mutation.id);\n lc = lc.withContext('processMutation');\n // The mutationID context set above identifies it; the mutation itself\n // carries the row values being written.\n lc.debug?.('Process mutation start');\n\n // Record mutation processing attempt for telemetry (regardless of success/failure)\n recordMutation('crud');\n\n let result: MutationError \| undefined;\n\n const start = Date.now();\n try {\n // Mutations can fail for a variety of reasons:\n //\n // - application error\n // - network/db error\n // - zero bug\n //\n // For application errors what we want is to re-run the mutation in\n // \"error mode\", which skips the actual mutation and just updates the\n // lastMutationID. Then return the error to the app.\n //\n // However, it's hard to tell the difference between application errors\n // and the other types.\n //\n // A reasonable policy ends up being to just retry every mutation once\n // in error mode. If the error mode mutation succeeds then we assume it\n // was an application error and return the error to the app. Otherwise,\n // we know it was something internal and we log it.\n //\n // This is not 100% correct - there are theoretical cases where we\n // return an internal error to the app that shouldn't have been. But it\n // would have to be a crazy coincidence: we'd have to have a network\n // error on the first attempt that resolves by the second attempt.\n //\n // One might ask why not try/catch just the calls to the mutators and\n // consider those application errors. That is actually what we do in\n // Replicache:\n //\n // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n //\n // We don't do it here because:\n //\n // 1. It's still not perfect. It's hard to isolate SQL errors in\n // mutators due to app developer mistakes from SQL errors due to\n // Zero mistakes.\n // 2. It's not possible to do this with the pg library we're using in\n // Zero anyway: https://github.com/porsager/postgres/issues/455.\n //\n // Personally I think this simple retry policy is nice.\n let errorMode = false;\n for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n try {\n await runTx(\n db,\n async tx => {\n // Simulates a concurrent request for testing. In production this is a noop.\n const done = onTxStart?.();\n try {\n return await processMutationWithTx(\n lc,\n tx,\n authData,\n shard,\n clientGroupID,\n mutation,\n errorMode,\n writeAuthorizer,\n );\n } finally {\n await done;\n }\n },\n {mode: Mode.SERIALIZABLE},\n );\n if (errorMode) {\n lc.debug?.('Ran mutation successfully in error mode');\n }\n break;\n } catch (e) {\n if (e instanceof MutationAlreadyProcessedError) {\n lc.debug?.(e.message);\n // Don't double-count already processed mutations, but they were counted above\n return undefined;\n }\n if (\n isProtocolError(e) &&\n !errorMode &&\n e.kind === ErrorKind.InvalidPush &&\n customMutatorsEnabled &&\n i < 2\n ) {\n // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n // This can create a lot of OOO mutation errors since we do not know when the API server\n // has applied a custom mutation before moving on to process CRUD mutations.\n // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n // They should migrate completely to custom mutators.\n lc.info?.(\n 'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n e,\n );\n await new Promise(resolve => setTimeout(resolve, 100));\n continue;\n }\n if (isProtocolError(e) \|\| errorMode) {\n lc.error?.('Process mutation error', e);\n throw e;\n }\n if (\n e instanceof postgres.PostgresError &&\n e.code === PG_SERIALIZATION_FAILURE\n ) {\n lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n }\n result = [ErrorKind.MutationFailed, String(e)];\n if (errorMode) {\n break;\n }\n lc.error?.('Got error running mutation, re-running in error mode', e);\n errorMode = true;\n i--;\n }\n }\n } finally {\n lc.debug?.('Process mutation complete in', Date.now() - start);\n }\n return result;\n}\n\nexport async function processMutationWithTx(\n lc: LogContext,\n tx: PostgresTransaction,\n authData: JWTPayload \| undefined,\n shard: ShardID,\n clientGroupID: string,\n mutation: CRUDMutation,\n errorMode: boolean,\n authorizer: WriteAuthorizer,\n) {\n const tasks: (() => Promise<unknown>)[] = [];\n\n async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n try {\n return await stmt.execute();\n } finally {\n const q = stmt as unknown as Query;\n // `q.parameters` are the bound row values -- log the parameterized\n // statement and the parameter count, never the values themselves.\n lc.debug?.(`${q.string} (${q.parameters.length} params)`);\n }\n }\n\n authorizer.reloadPermissions();\n\n if (!errorMode) {\n // Validates table names and keys every op by the replica's primary key,\n // which is what the authorizer checks.\n const ops = authorizer.validateOps(mutation.args[0].ops);\n\n const normalizedOps = authorizer.normalizeOps(ops);\n const [canPre, canPost] = await Promise.all([\n authorizer.canPreMutation(authData, normalizedOps),\n authorizer.canPostMutation(authData, normalizedOps),\n ]);\n if (canPre && canPost) {\n for (const op of ops) {\n switch (op.op) {\n case 'insert':\n tasks.push(() => execute(getInsertSQL(tx, op)));\n break;\n case 'upsert':\n tasks.push(() => execute(getUpsertSQL(tx, op)));\n break;\n case 'update':\n tasks.push(() => execute(getUpdateSQL(tx, op)));\n break;\n case 'delete':\n tasks.push(() => execute(getDeleteSQL(tx, op)));\n break;\n default:\n unreachable(op);\n }\n }\n }\n }\n\n // Confirm the mutation even though it may have been blocked by the authorizer.\n // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n tasks.unshift(() =>\n checkSchemaVersionAndIncrementLastMutationID(\n tx,\n shard,\n clientGroupID,\n mutation.clientID,\n mutation.id,\n ),\n );\n\n // Note: An error thrown from any Promise aborts the entire transaction.\n await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n tx: postgres.TransactionSql,\n create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n return tx`\n INSERT INTO ${tx(create.tableName)} ${tx(create.value)}\n ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING\n `;\n}\n\nexport function getUpsertSQL(\n tx: postgres.TransactionSql,\n set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const {tableName, primaryKey, value} = set;\n return tx`\n INSERT INTO ${tx(tableName)} ${tx(value)}\n ON CONFLICT (${tx(primaryKey)})\n DO UPDATE SET ${tx(value)}\n `;\n}\n\nfunction getUpdateSQL(\n tx: postgres.TransactionSql,\n update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const table = update.tableName;\n const {primaryKey, value} = update;\n const id: Record<string, PrimaryKeyValue> = {};\n for (const key of primaryKey) {\n id[key] = v.parse(value[key], primaryKeyValueSchema);\n }\n return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n id,\n ).flatMap(([key, value], i) =>\n i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n )}`;\n}\n\nfunction getDeleteSQL(\n tx: postgres.TransactionSql,\n deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const {tableName, primaryKey, value} = deleteOp;\n\n const conditions = [];\n for (const key of primaryKey) {\n if (conditions.length > 0) {\n conditions.push(tx`AND`);\n }\n conditions.push(tx`${tx(key)} = ${value[key]}`);\n }\n\n return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n tx: PostgresTransaction,\n shard: ShardID,\n clientGroupID: string,\n clientID: string,\n receivedMutationID: number,\n) {\n const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n INSERT INTO ${tx(upstreamSchema(shard))}.clients \n as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n VALUES (${clientGroupID}, ${clientID}, ${1})\n ON CONFLICT (\"clientGroupID\", \"clientID\")\n DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n RETURNING \"lastMutationID\"\n `;\n\n // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n if (receivedMutationID < lastMutationID) {\n throw new MutationAlreadyProcessedError(\n clientID,\n receivedMutationID,\n lastMutationID,\n );\n } else if (receivedMutationID > lastMutationID) {\n throw new ProtocolError({\n kind: ErrorKind.InvalidPush,\n message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n origin: ErrorOrigin.ZeroCache,\n });\n }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAK,MAAM;EACX,KAAK,YAAY;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,SAAS,KAAK,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAK,mBAAmB,IAAI,oBAC1B,KAAK,KACL,QACA,KAAK,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAK,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;EACP,IAAI,KAAK,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAK,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAK,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrBA,qBACA,qBACF,CAAC;EAEH,KAAK,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAK,KACL,UACA,KAAK,WACL,KAAK,QACL,KAAK,IACL,UACA,KAAK,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAK,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAK,YACP,OAAO,KAAK,SAAS;EAEvB,KAAK,aAAa;EAClB,IAAI;GACF,KAAK,iBAAiB,QAAQ;EAChC,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,6CAA6C,CAAC;EACjE,UAAU;GAIR,KAAK,SAAS,MAAM,EAAC,UAAU,MAAK,CAAC;EACvC;EACA,KAAK,SAAS,QAAQ;EACtB,OAAO,KAAK,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAASC,MAClB,mCACF;CAEA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CAGrC,GAAG,QAAQ,wBAAwB;CAGnC,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAMC,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAAQ,EAAE,OAAO;IAEpB;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAACC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GAGV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,EAAE,WAAW,OAAO,SAAS;EAC1D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EAGd,MAAM,MAAM,WAAW,YAAY,SAAS,KAAK,EAAE,CAAC,GAAG;EAEvD,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAgB,aACd,IACA,QACuC;CACvC,OAAO,EAAE;kBACO,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK,EAAE;mBACxC,GAAG,OAAO,UAAU,EAAE;;AAEzC;AAEA,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAOC,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAMC;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQC;CACV,CAAC;AAEL"}

package/out/zero-cache/src/services/mutagen/pusher.js CHANGED Viewed

@@ -167,6 +167,18 @@ var PushWorker = class {
 	#lc;
 	#config;
 	#clients;
+	/**
+	* For each client, the wsID of its socket whose push most recently failed.
+	* Pushes queued behind the failed one on that socket are not sent: they
+	* start past the failed mutation, so the API server could only reject them
+	* as out of order. The socket is failed (or was already replaced), and the
+	* client re-sends every unacknowledged mutation on its next socket.
+	*
+	* An entry is kept until the socket can no longer queue pushes (it closed or
+	* the client reconnected) and the queue has been drained since, so it is not
+	* removed while pushes from the failed socket may still be queued.
+	*/
+	#failedSockets = /* @__PURE__ */ new Map();
 	#customMutations = getOrCreateCounter("mutation", "custom", "Number of custom mutations processed");
 	#pushes = getOrCreateCounter("mutation", "pushes", "Number of pushes processed by the pusher");
 	constructor(config, lc, connContextManager, queue) {
@@ -201,28 +213,47 @@ var PushWorker = class {
 	}
 	async run() {
 		for (;;) {
+			const closedSockets = [...this.#failedSockets].filter(([clientID, wsID]) => !this.#connContextManager.getConnectionContext({
+				clientID,
+				wsID
+			}));
 			const [pushes, terminate] = combinePushes([await this.#queue.dequeue(), ...this.#queue.drain()]);
 			for (const push of pushes) {
+				const { clientID, wsID } = push.connCtx;
+				if (this.#failedSockets.get(clientID) === wsID) {
+					this.#lc.debug?.("Not sending push queued behind a failed push on the same socket; the client re-sends it on its next socket", {
+						clientID,
+						wsID,
+						mutations: push.push.mutations.length
+					});
+					continue;
+				}
 				const parentContext = push.push.traceparent ? propagation.extract(ROOT_CONTEXT, { traceparent: push.push.traceparent }) : context.active();
 				const response = await context.with(parentContext, () => this.#processPush(push));
-				await this.#fanOutResponses(response);
+				await this.#fanOutResponses(push.connCtx, response);
 			}
+			for (const [clientID, wsID] of closedSockets) if (this.#failedSockets.get(clientID) === wsID) this.#failedSockets.delete(clientID);
 			if (terminate) break;
 		}
 	}
 	/**
-	* 1. If the entire `push` fails, we send the error to relevant clients.
+	* 1. If the entire `push` fails, we send the error to `origin`, the
+	*  connection the push was sent on.
 	* 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate
-	*  and terminate the connection for those clients.
+	*  and terminate `origin`.
+	*
+	* Only `origin` is failed, even when the push carried other clients'
+	* mutations (the client group's commit chain includes other tabs' pending
+	* mutations). A client tracks what it has sent per socket, so the gap a
+	* failed push leaves is on `origin` alone. Other clients send their own
+	* mutations on their own sockets, and see any lasting error there.
 	*/
-	#fanOutResponses(response) {
+	#fanOutResponses(origin, response) {
 		const connectionTerminations = [];
 		if ("kind" in response && response.kind === "PushFailed" || "error" in response) {
 			this.#lc.warn?.("The server behind ZERO_MUTATE_URL returned a push error.", response);
-			const groupedMutationIDs = groupBy(response.mutationIDs ?? [], (m) => m.clientID);
-			for (const [clientID, mutationIDs] of groupedMutationIDs) {
-				const client = this.#clients.get(clientID);
-				if (!client) continue;
+			const mutationIDs = response.mutationIDs ?? [];
+			if (mutationIDs.length > 0) {
 				if ("error" in response) {
 					const pushFailedBody = response.error === "http" ? {
 						kind: PushFailed,
@@ -245,14 +276,12 @@ var PushWorker = class {
 						mutationIDs,
 						message: response.error === "zeroPusher" ? response.details : response.error === "unsupportedSchemaVersion" ? "Unsupported schema version" : "An unknown error occurred while pushing to the API server"
 					};
-					this.#failDownstream(client.downstream, pushFailedBody);
-				} else this.#failDownstream(client.downstream, response);
+					this.#failOrigin(origin, pushFailedBody);
+				} else this.#failOrigin(origin, response);
 			}
 		} else {
 			const groupedMutations = groupBy(response.mutations, (m) => m.id.clientID);
-			for (const [clientID, mutations] of groupedMutations) {
-				const client = this.#clients.get(clientID);
-				if (!client) continue;
+			for (const mutations of groupedMutations.values()) {
 				let failure;
 				let i = 0;
 				for (; i < mutations.length; i++) {
@@ -274,11 +303,25 @@ var PushWorker = class {
 					}
 				}
 				if (failure && i < mutations.length - 1) this.#lc.warn?.("push-response contains mutations after a mutation which should fatal the connection");
-				if (failure) connectionTerminations.push(() => this.#failDownstream(client.downstream, failure));
+				if (failure) connectionTerminations.push(() => this.#failOrigin(origin, failure));
 			}
 		}
 		connectionTerminations.forEach((cb) => cb());
 	}
+	/**
+	* Fails `origin` after a push sent on it failed, and records it in
+	* `#failedSockets` so the pushes it queued behind that one are not sent.
+	*
+	* If the client has reconnected since sending the push, the old socket is
+	* already gone and the client re-sends its mutations on the new one. Failing
+	* the new socket would put the client in its error state, rejecting its
+	* outstanding mutations, for a failure it has already recovered from.
+	*/
+	#failOrigin(origin, errorBody) {
+		this.#failedSockets.set(origin.clientID, origin.wsID);
+		const client = this.#clients.get(origin.clientID);
+		if (client?.wsID === origin.wsID) this.#failDownstream(client.downstream, errorBody);
+	}
 	async #processPush(entry) {
 		this.#customMutations.add(entry.push.mutations.length, { clientGroupID: entry.push.clientGroupID });
 		this.#pushes.add(1, { clientGroupID: entry.push.clientGroupID });