npm - @rocicorp/zero - Versions diffs - 1.5.0-canary.3 → 1.6.0-canary.0 - Mend

@rocicorp/zero 1.5.0-canary.3 → 1.6.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (172) hide show

package/out/zero-cache/src/services/change-source/custom/change-source.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n AutoResetSignal,\n type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n createReplicationStateTables,\n getSubscriptionState,\n initReplicationState,\n type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n type BackfillRequest,\n type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\n/\n Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n /\nexport async function initializeCustomChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n context: ServerContext,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n (log, tx) => initialSync(log, shard, tx, upstreamURI, context),\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n if (shard.publications.length) {\n // Verify that the publications match what has been synced.\n const requested = shard.publications.toSorted();\n const replicated = subscriptionState.publications.sort();\n if (!deepEqual(requested, replicated)) {\n throw new Error(\n `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n );\n }\n }\n\n const changeSource = new CustomChangeSource(\n lc,\n upstreamURI,\n shard,\n subscriptionState,\n );\n\n return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replicationConfig: ReplicationConfig;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replicationConfig: ReplicationConfig,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replicationConfig = replicationConfig;\n }\n\n initialSync(): ChangeStream {\n return this.#startStream();\n }\n\n startLagReporter() {\n return null; // Not supported for custom sources\n }\n\n stop(): Promise<void> {\n return Promise.resolve();\n }\n\n startStream(\n clientWatermark: string,\n backfillRequests: BackfillRequest[] = [],\n ): Promise<ChangeStream> {\n if (backfillRequests?.length) {\n throw new Error(\n 'backfill is yet not supported for custom change sources',\n );\n }\n return Promise.resolve(this.#startStream(clientWatermark));\n }\n\n #startStream(clientWatermark?: string): ChangeStream {\n const {publications, replicaVersion} = this.#replicationConfig;\n const {appID, shardNum} = this.#shard;\n const url = new URL(this.#upstreamUri);\n url.searchParams.set('appID', appID);\n url.searchParams.set('shardNum', String(shardNum));\n for (const pub of publications) {\n url.searchParams.append('publications', pub);\n }\n if (clientWatermark) {\n assert(\n replicaVersion.length,\n 'replicaVersion is required when clientWatermark is set',\n );\n url.searchParams.set('lastWatermark', clientWatermark);\n url.searchParams.set('replicaVersion', replicaVersion);\n }\n\n const ws = new WebSocket(url);\n const {instream, outstream} = stream(\n this.#lc,\n ws,\n changeStreamMessageSchema,\n // Upstream acks coalesce. If upstream exhibits back-pressure,\n // only the last ACK is kept / buffered.\n {coalesce: (curr: ChangeSourceUpstream) => curr},\n );\n return {changes: instream, acks: outstream};\n }\n}\n\n/\n Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n \n Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n context: ServerContext,\n) {\n const {appID: id, publications} = shard;\n const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n replicaVersion: '', // ignored for initialSync()\n publications,\n });\n const {changes} = changeSource.initialSync();\n\n createReplicationStateTables(tx);\n const processor = new ChangeProcessor(\n new StatementRunner(tx),\n 'initial-sync',\n (_, err) => {\n throw err;\n },\n );\n\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n try {\n let num = 0;\n for await (const change of changes) {\n const [tag] = change;\n switch (tag) {\n case 'begin': {\n const {commitWatermark} = change[2];\n lc.info?.(\n `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying upstream tables at version ${commitWatermark}`,\n 5000,\n );\n initReplicationState(\n tx,\n publications.toSorted(),\n commitWatermark,\n context,\n false,\n );\n processor.processMessage(lc, change);\n break;\n }\n case 'data':\n processor.processMessage(lc, change);\n if (++num % 1000 === 0) {\n lc.debug?.(`processed ${num} changes`);\n }\n break;\n case 'commit':\n processor.processMessage(lc, change);\n validateInitiallySyncedData(lc, tx, shard);\n lc.info?.(`finished initial-sync of ${num} changes`);\n return;\n\n case 'status':\n break; // Ignored\n // @ts-expect-error: falls through if the tag is not 'reset-required\n case 'control': {\n const {tag, message} = change[1];\n if (tag === 'reset-required') {\n throw new AutoResetSignal(\n message ?? 'auto-reset signaled by change source',\n );\n }\n }\n // falls through\n case 'rollback':\n throw new Error(\n `unexpected message during initial-sync: ${stringify(change)}`,\n );\n default:\n unreachable(change);\n }\n }\n throw new Error(\n `change source ${upstreamURI} closed before initial-sync completed`,\n );\n } catch (e) {\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n appID,\n shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n return {\n [`${appID}_${shardNum}.clients`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n lastMutationID: {type: 'number'},\n userID: {type: 'string'},\n },\n [`${appID}_${shardNum}.mutations`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n mutationID: {type: 'number'},\n mutation: {type: 'json'},\n },\n [`${appID}.permissions`]: {\n permissions: {type: 'json'},\n hash: {type: 'string'},\n },\n };\n}\n\nfunction validateInitiallySyncedData(\n lc: LogContext,\n db: Database,\n shard: ShardID,\n) {\n const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n const required = getRequiredTables(shard);\n for (const [name, columns] of Object.entries(required)) {\n const table = tables.get(name)?.zqlSpec;\n if (!table) {\n throw new Error(\n `Upstream is missing the \"${name}\" table. (Found ${[\n ...tables.keys(),\n ]})` +\n `Please ensure that each table has a unique index over one ` +\n `or more non-null columns.`,\n );\n }\n for (const [col, {type}] of Object.entries(columns)) {\n const found = table[col];\n if (!found) {\n throw new Error(\n `Upstream \"${table}\" table is missing the \"${col}\" column`,\n );\n }\n if (found.type !== type) {\n throw new Error(\n `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SAC6E;AAC7E,OAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,QAAQ,CAC/D;CAED,MAAM,UAAU,IAAI,SAAS,IAAI,cAAc;CAC/C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,QAAQ,CAAC;AAC5E,SAAQ,OAAO;AAEf,KAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,UAAU;EAC/C,MAAM,aAAa,kBAAkB,aAAa,MAAM;AACxD,MAAI,CAAC,UAAU,WAAW,WAAW,CACnC,OAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,GAC7G;;AAWL,QAAO;EAAC;EAAmB,cAPN,IAAI,mBACvB,IACA,aACA,OACA,kBACD;EAEuC;;AAG1C,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,cAAoB;AACpB,QAAA,QAAc;AACd,QAAA,oBAA0B;;CAG5B,cAA4B;AAC1B,SAAO,MAAA,aAAmB;;CAG5B,mBAAmB;AACjB,SAAO;;CAGT,OAAsB;AACpB,SAAO,QAAQ,SAAS;;CAG1B,YACE,iBACA,mBAAsC,EAAE,EACjB;AACvB,MAAI,kBAAkB,OACpB,OAAM,IAAI,MACR,0DACD;AAEH,SAAO,QAAQ,QAAQ,MAAA,YAAkB,gBAAgB,CAAC;;CAG5D,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,MAAA;EACvC,MAAM,EAAC,OAAO,aAAY,MAAA;EAC1B,MAAM,MAAM,IAAI,IAAI,MAAA,YAAkB;AACtC,MAAI,aAAa,IAAI,SAAS,MAAM;AACpC,MAAI,aAAa,IAAI,YAAY,OAAO,SAAS,CAAC;AAClD,OAAK,MAAM,OAAO,aAChB,KAAI,aAAa,OAAO,gBAAgB,IAAI;AAE9C,MAAI,iBAAiB;AACnB,UACE,eAAe,QACf,yDACD;AACD,OAAI,aAAa,IAAI,iBAAiB,gBAAgB;AACtD,OAAI,aAAa,IAAI,kBAAkB,eAAe;;EAGxD,MAAM,KAAK,IAAI,UAAU,IAAI;EAC7B,MAAM,EAAC,UAAU,cAAa,OAC5B,MAAA,IACA,IACA,2BAGA,EAAC,WAAW,SAA+B,MAAK,CACjD;AACD,SAAO;GAAC,SAAS;GAAU,MAAM;GAAU;;;;;;;;;;;;;;AAe/C,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAJc,IAAI,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;EACD,CAAC,CAC6B,aAAa;AAE5C,8BAA6B,GAAG;CAChC,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,GAAG,EACvB,iBACC,GAAG,QAAQ;AACV,QAAM;GAET;CAED,MAAM,kBAAkB,2BAA2B,sBAAsB,GAAG;AAC5E,KAAI;EACF,IAAI,MAAM;AACV,aAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;AACd,WAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;AACjC,QAAG,OACD,yBAAyB,GAAG,qBAAqB,kBAClD;AACD,qBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,IACD;AACD,0BACE,IACA,aAAa,UAAU,EACvB,iBACA,SACA,MACD;AACD,eAAU,eAAe,IAAI,OAAO;AACpC;;IAEF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,SAAI,EAAE,MAAM,QAAS,EACnB,IAAG,QAAQ,aAAa,IAAI,UAAU;AAExC;IACF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,iCAA4B,IAAI,IAAI,MAAM;AAC1C,QAAG,OAAO,4BAA4B,IAAI,UAAU;AACpD;IAEF,KAAK,SACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;AAC9B,SAAI,QAAQ,iBACV,OAAM,IAAI,gBACR,WAAW,uCACZ;;IAIL,KAAK,WACH,OAAM,IAAI,MACR,2CAA2C,UAAU,OAAO,GAC7D;IACH,QACE,aAAY,OAAO;;;AAGzB,QAAM,IAAI,MACR,iBAAiB,YAAY,uCAC9B;UACM,GAAG;AACV,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;;;AAM1B,SAAS,kBAAkB,EACzB,OACA,YACuD;AACvD,QAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,gBAAgB,EAAC,MAAM,UAAS;GAChC,QAAQ,EAAC,MAAM,UAAS;GACzB;GACA,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,YAAY,EAAC,MAAM,UAAS;GAC5B,UAAU,EAAC,MAAM,QAAO;GACzB;GACA,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,QAAO;GAC3B,MAAM,EAAC,MAAM,UAAS;GACvB;EACF;;AAGH,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,MAAK,CAAC;CACzE,MAAM,WAAW,kBAAkB,MAAM;AACzC,MAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,SAAS,EAAE;EACtD,MAAM,QAAQ,OAAO,IAAI,KAAK,EAAE;AAChC,MAAI,CAAC,MACH,OAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,MAAM,CACjB,CAAC,sFAGH;AAEH,OAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,QAAQ,EAAE;GACnD,MAAM,QAAQ,MAAM;AACpB,OAAI,CAAC,MACH,OAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,UAClD;AAEH,OAAI,MAAM,SAAS,KACjB,OAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,QACjF"}
1	+ {"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n AutoResetSignal,\n type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n createReplicationStateTables,\n getSubscriptionState,\n initReplicationState,\n type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n type BackfillRequest,\n type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\n/\n Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n /\nexport async function initializeCustomChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n context: ServerContext,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n (log, tx) => initialSync(log, shard, tx, upstreamURI, context),\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n if (shard.publications.length) {\n // Verify that the publications match what has been synced.\n const requested = shard.publications.toSorted();\n const replicated = subscriptionState.publications.sort();\n if (!deepEqual(requested, replicated)) {\n throw new Error(\n `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n );\n }\n }\n\n const changeSource = new CustomChangeSource(\n lc,\n upstreamURI,\n shard,\n subscriptionState,\n );\n\n return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replicationConfig: ReplicationConfig;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replicationConfig: ReplicationConfig,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replicationConfig = replicationConfig;\n }\n\n initialSync(): ChangeStream {\n return this.#startStream();\n }\n\n startLagReporter() {\n return null; // Not supported for custom sources\n }\n\n stop(): Promise<void> {\n return Promise.resolve();\n }\n\n startStream(\n clientWatermark: string,\n backfillRequests: BackfillRequest[] = [],\n ): Promise<ChangeStream> {\n if (backfillRequests?.length) {\n throw new Error(\n 'backfill is yet not supported for custom change sources',\n );\n }\n return Promise.resolve(this.#startStream(clientWatermark));\n }\n\n #startStream(clientWatermark?: string): ChangeStream {\n const {publications, replicaVersion} = this.#replicationConfig;\n const {appID, shardNum} = this.#shard;\n const url = new URL(this.#upstreamUri);\n url.searchParams.set('appID', appID);\n url.searchParams.set('shardNum', String(shardNum));\n for (const pub of publications) {\n url.searchParams.append('publications', pub);\n }\n if (clientWatermark) {\n assert(\n replicaVersion.length,\n 'replicaVersion is required when clientWatermark is set',\n );\n url.searchParams.set('lastWatermark', clientWatermark);\n url.searchParams.set('replicaVersion', replicaVersion);\n }\n\n const ws = new WebSocket(url);\n const {instream, outstream} = stream(\n this.#lc,\n ws,\n changeStreamMessageSchema,\n // Upstream acks coalesce. If upstream exhibits back-pressure,\n // only the last ACK is kept / buffered.\n {coalesce: (curr: ChangeSourceUpstream) => curr},\n );\n return {changes: instream, acks: outstream};\n }\n}\n\n/\n Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n \n Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n context: ServerContext,\n) {\n const {appID: id, publications} = shard;\n const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n replicaVersion: '', // ignored for initialSync()\n publications,\n });\n const {changes} = changeSource.initialSync();\n\n createReplicationStateTables(tx);\n const processor = new ChangeProcessor(\n new StatementRunner(tx),\n 'initial-sync',\n (_, err) => {\n throw err;\n },\n );\n\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n try {\n let num = 0;\n for await (const change of changes) {\n const [tag] = change;\n switch (tag) {\n case 'begin': {\n const {commitWatermark} = change[2];\n lc.info?.(\n `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying upstream tables at version ${commitWatermark}`,\n 5000,\n );\n initReplicationState(\n tx,\n publications.toSorted(),\n commitWatermark,\n context,\n false,\n );\n processor.processMessage(lc, change);\n break;\n }\n case 'data':\n processor.processMessage(lc, change);\n if (++num % 1000 === 0) {\n lc.debug?.(`processed ${num} changes`);\n }\n break;\n case 'commit':\n processor.processMessage(lc, change);\n validateInitiallySyncedData(lc, tx, shard);\n lc.info?.(`finished initial-sync of ${num} changes`);\n return;\n\n case 'status':\n break; // Ignored\n // @ts-expect-error: falls through if the tag is not 'reset-required\n case 'control': {\n const {tag, message} = change[1];\n if (tag === 'reset-required') {\n throw new AutoResetSignal(\n message ?? 'auto-reset signaled by change source',\n );\n }\n }\n // falls through\n case 'rollback':\n throw new Error(\n `unexpected message during initial-sync: ${stringify(change)}`,\n );\n default:\n unreachable(change);\n }\n }\n throw new Error(\n `change source ${upstreamURI} closed before initial-sync completed`,\n );\n } catch (e) {\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n appID,\n shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n return {\n [`${appID}_${shardNum}.clients`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n lastMutationID: {type: 'number'},\n userID: {type: 'string'},\n },\n [`${appID}_${shardNum}.mutations`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n mutationID: {type: 'number'},\n mutation: {type: 'json'},\n },\n [`${appID}.permissions`]: {\n permissions: {type: 'json'},\n hash: {type: 'string'},\n },\n };\n}\n\nfunction validateInitiallySyncedData(\n lc: LogContext,\n db: Database,\n shard: ShardID,\n) {\n const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n const required = getRequiredTables(shard);\n for (const [name, columns] of Object.entries(required)) {\n const table = tables.get(name)?.zqlSpec;\n if (!table) {\n throw new Error(\n `Upstream is missing the \"${name}\" table. (Found ${[\n ...tables.keys(),\n ]})` +\n `Please ensure that each table has a unique index over one ` +\n `or more non-null columns.`,\n );\n }\n for (const [col, {type}] of Object.entries(columns)) {\n const found = table[col];\n if (!found) {\n throw new Error(\n `Upstream \"${table}\" table is missing the \"${col}\" column`,\n );\n }\n if (found.type !== type) {\n throw new Error(\n `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SAC6E;AAC7E,OAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,QAAQ,CAC/D;CAED,MAAM,UAAU,IAAI,SAAS,IAAI,cAAc;CAC/C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,QAAQ,CAAC;AAC5E,SAAQ,OAAO;AAEf,KAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,UAAU;EAC/C,MAAM,aAAa,kBAAkB,aAAa,MAAM;AACxD,MAAI,CAAC,UAAU,WAAW,WAAW,CACnC,OAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,GAC7G;;AAWL,QAAO;EAAC;EAAmB,cAPN,IAAI,mBACvB,IACA,aACA,OACA,kBACD;EAEuC;;AAG1C,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,cAAoB;AACpB,QAAA,QAAc;AACd,QAAA,oBAA0B;;CAG5B,cAA4B;AAC1B,SAAO,MAAA,aAAmB;;CAG5B,mBAAmB;AACjB,SAAO;;CAGT,OAAsB;AACpB,SAAO,QAAQ,SAAS;;CAG1B,YACE,iBACA,mBAAsC,EAAE,EACjB;AACvB,MAAI,kBAAkB,OACpB,OAAM,IAAI,MACR,0DACD;AAEH,SAAO,QAAQ,QAAQ,MAAA,YAAkB,gBAAgB,CAAC;;CAG5D,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,MAAA;EACvC,MAAM,EAAC,OAAO,aAAY,MAAA;EAC1B,MAAM,MAAM,IAAI,IAAI,MAAA,YAAkB;AACtC,MAAI,aAAa,IAAI,SAAS,MAAM;AACpC,MAAI,aAAa,IAAI,YAAY,OAAO,SAAS,CAAC;AAClD,OAAK,MAAM,OAAO,aAChB,KAAI,aAAa,OAAO,gBAAgB,IAAI;AAE9C,MAAI,iBAAiB;AACnB,UACE,eAAe,QACf,yDACD;AACD,OAAI,aAAa,IAAI,iBAAiB,gBAAgB;AACtD,OAAI,aAAa,IAAI,kBAAkB,eAAe;;EAGxD,MAAM,KAAK,IAAI,YAAU,IAAI;EAC7B,MAAM,EAAC,UAAU,cAAa,OAC5B,MAAA,IACA,IACA,2BAGA,EAAC,WAAW,SAA+B,MAAK,CACjD;AACD,SAAO;GAAC,SAAS;GAAU,MAAM;GAAU;;;;;;;;;;;;;;AAe/C,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAJc,IAAI,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;EACD,CAAC,CAC6B,aAAa;AAE5C,8BAA6B,GAAG;CAChC,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,GAAG,EACvB,iBACC,GAAG,QAAQ;AACV,QAAM;GAET;CAED,MAAM,kBAAkB,2BAA2B,sBAAsB,GAAG;AAC5E,KAAI;EACF,IAAI,MAAM;AACV,aAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;AACd,WAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;AACjC,QAAG,OACD,yBAAyB,GAAG,qBAAqB,kBAClD;AACD,qBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,IACD;AACD,0BACE,IACA,aAAa,UAAU,EACvB,iBACA,SACA,MACD;AACD,eAAU,eAAe,IAAI,OAAO;AACpC;;IAEF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,SAAI,EAAE,MAAM,QAAS,EACnB,IAAG,QAAQ,aAAa,IAAI,UAAU;AAExC;IACF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,iCAA4B,IAAI,IAAI,MAAM;AAC1C,QAAG,OAAO,4BAA4B,IAAI,UAAU;AACpD;IAEF,KAAK,SACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;AAC9B,SAAI,QAAQ,iBACV,OAAM,IAAI,gBACR,WAAW,uCACZ;;IAIL,KAAK,WACH,OAAM,IAAI,MACR,2CAA2C,UAAU,OAAO,GAC7D;IACH,QACE,aAAY,OAAO;;;AAGzB,QAAM,IAAI,MACR,iBAAiB,YAAY,uCAC9B;UACM,GAAG;AACV,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;;;AAM1B,SAAS,kBAAkB,EACzB,OACA,YACuD;AACvD,QAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,gBAAgB,EAAC,MAAM,UAAS;GAChC,QAAQ,EAAC,MAAM,UAAS;GACzB;GACA,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,YAAY,EAAC,MAAM,UAAS;GAC5B,UAAU,EAAC,MAAM,QAAO;GACzB;GACA,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,QAAO;GAC3B,MAAM,EAAC,MAAM,UAAS;GACvB;EACF;;AAGH,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,MAAK,CAAC;CACzE,MAAM,WAAW,kBAAkB,MAAM;AACzC,MAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,SAAS,EAAE;EACtD,MAAM,QAAQ,OAAO,IAAI,KAAK,EAAE;AAChC,MAAI,CAAC,MACH,OAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,MAAM,CACjB,CAAC,sFAGH;AAEH,OAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,QAAQ,EAAE;GACnD,MAAM,QAAQ,MAAM;AACpB,OAAI,CAAC,MACH,OAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,UAClD;AAEH,OAAI,MAAM,SAAS,KACjB,OAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,QACjF"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js CHANGED Viewed

@@ -11,7 +11,8 @@ import { TsvParser } from "../../../db/pg-copy.js";
 import { getTypeParsers } from "../../../db/pg-type-parser.js";
 import { TransactionPool, importSnapshot } from "../../../db/transaction-pool.js";
 import { columnMetadataSchema, tableMetadataSchema } from "./backfill-metadata.js";
-import { createReplicationSlot, makeBinarySelectExprs, makeDownloadStatements } from "./initial-sync.js";
+import { createReplicationSlot } from "./replication-slots.js";
+import { makeBinarySelectExprs, makeDownloadStatements } from "./initial-sync.js";
 import postgres from "postgres";
 import { PG_UNDEFINED_COLUMN, PG_UNDEFINED_TABLE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/backfill-stream.ts
@@ -136,16 +137,16 @@ async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
 		["fetch_types"]: false,
 		connection: { replication: "database" }
 	});
-	const tempSlot = `${slotNamePrefix}_bf_${Date.now()}`;
+	const slotName = `${slotNamePrefix}_bf_${Date.now()}`;
 	try {
-		const { snapshot_name: snapshot, consistent_point: lsn } = await createReplicationSlot(lc, replicationSession, tempSlot);
+		const { snapshot_name: snapshot, consistent_point: lsn } = await createReplicationSlot(lc, replicationSession, { slotName });
 		const { init, imported } = importSnapshot(snapshot);
 		const tx = new TransactionPool(lc, {
 			mode: READONLY,
 			init
 		}).run(db);
 		await imported;
-		await replicationSession.unsafe(`DROP_REPLICATION_SLOT "${tempSlot}"`);
+		await replicationSession.unsafe(`DROP_REPLICATION_SLOT "${slotName}"`);
 		const watermark = toStateVersionString(lsn);
 		lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);
 		return {
@@ -154,8 +155,8 @@ async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
 		};
 	} catch (e) {
 		await replicationSession.unsafe(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots
-         WHERE slot_name = '${tempSlot}'`);
-		lc.error?.(`Failed to create backfill snapshot`, e);
+         WHERE slot_name = '${slotName}'`);
+		lc.warn?.(`Failed to create backfill snapshot`, e);
 		throw e;
 	} finally {
 		await replicationSession.end();

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n DownloadStatus,\n JSONValue,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n createReplicationSlot,\n makeBinarySelectExprs,\n makeDownloadStatements,\n type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n /*\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number \| undefined;\n\n /\n Use text-format COPY instead of binary COPY.\n * Binary is faster and handles all types (unknown types are cast to\n * `::text` in the SELECT). This flag exists as an escape hatch to\n * revert to the old code path if needed.\n /\n textCopy?: boolean \| undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b\|\\sLIMIT\\s+\\d/i;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n opts;\n const db = pgClient(lc, upstreamURI, 'backfill-stream', {\n ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n });\n let tx: TransactionPool \| undefined;\n let watermark: string;\n try {\n ({tx, watermark} = await createSnapshotTransaction(\n lc,\n upstreamURI,\n db,\n slot,\n ));\n const {tableSpec, backfill} = await validateSchema(\n tx,\n publications,\n bf,\n watermark,\n );\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n const stmts = makeDownloadStatements(tableSpec, cols);\n\n if (textCopy) {\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${stmts.select}) TO STDOUT`,\n new TsvParser(),\n cols.map(col => {\n const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n return (text: string) => parser(text) as JSONValue;\n }),\n flushThresholdBytes,\n );\n } else {\n const binaryStmts = makeDownloadStatements(\n tableSpec,\n cols,\n undefined,\n undefined,\n makeBinarySelectExprs(tableSpec, cols),\n );\n\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n new BinaryCopyParser(),\n cols.map(col => {\n const spec = tableSpec.columns[col];\n const decoder = hasBinaryDecoder(spec)\n ? makeBinaryDecoder(spec)\n : textCastDecoder;\n return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n }),\n flushThresholdBytes,\n );\n }\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx?.setDone();\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream<T>(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n {\n getTotalRows,\n getTotalBytes,\n }: Pick<DownloadStatements, 'getTotalRows' \| 'getTotalBytes'>,\n copyCommand: string,\n parser: {parse(chunk: Buffer): Iterable<T \| null>},\n decoders: ((field: T) => JSONValue)[],\n flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n // sync and must never appear in a backfill COPY.\n assert(\n !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n `backfill COPY must not sample or limit: ${copyCommand}`,\n );\n const start = performance.now();\n const [rows, bytes] = await tx.processReadTask(sql =>\n Promise.all([\n sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n ]),\n );\n const status: DownloadStatus = {\n rows: 0,\n totalRows: Number(rows[0].totalRows),\n totalBytes: Number(bytes[0].totalBytes),\n };\n\n let elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n {\n status,\n },\n );\n const copyStream = await tx.processReadTask(sql =>\n sql.unsafe(copyCommand).readable(),\n );\n\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: decoders.length});\n let col = 0;\n\n for await (const data of copyStream) {\n const chunk = data as Buffer;\n for (const field of parser.parse(chunk)) {\n row[col] = field === null ? null : decoders[col](field);\n\n if (++col === decoders.length) {\n rowValues.push(row);\n status.rows++;\n row = Array.from({length: decoders.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n }\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n }\n\n yield {tag: 'backfill-completed', ...backfill, status};\n elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n /\nasync function createSnapshotTransaction(\n lc: LogContext,\n upstreamURI: string,\n db: PostgresDB,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(\n lc,\n upstreamURI,\n 'backfill-replication-session',\n {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n },\n );\n const tempSlot = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, tempSlot);\n\n const {init, imported} = importSnapshot(snapshot);\n const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n await imported;\n await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${tempSlot}\"`);\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return {tx, watermark};\n } catch (e) {\n // In the event of a failure, clean up the replication slot if created.\n await replicationSession.unsafe(\n /sql*/\n `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name = '${tempSlot}'`,\n );\n lc.error?.(`Failed to create backfill snapshot`, e);\n throw e;\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n return {tableSpec: spec, backfill};\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAgEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,EAAE,EAC6B;AACrD,MAAK,GACF,YAAY,aAAa,WAAW,CACpC,YAAY,SAAS,GAAG,MAAM,KAAK;CAEtC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,SAAS,IAAI,aAAa,mBAAmB,GACrD,iBAAiB,MACnB,CAAC;CACF,IAAI;CACJ,IAAI;AACJ,KAAI;AACF,GAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,KACD;EACD,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,UACD;EAGD,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,QAAQ;EACrD,MAAM,QAAQ,uBAAuB,WAAW,KAAK;AAErD,MAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK,CAAC;AAClE,UAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,WAAW,EACf,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,QAAQ;AAClE,YAAQ,SAAiB,OAAO,KAAK;KACrC,EACF,oBACD;SACI;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,KAAK,CACvC;AAED,UAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,kBAAkB,EACtB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,KAAK,GAClC,kBAAkB,KAAK,GACvB;AACJ,YAAQ,QAAgB,QAAQ,IAAI;KACpC,EACF,oBACD;;UAEI,GAAG;AASV,MACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,qBAE7C,OAAM,IAAI,2BAA2B,IAAI,OAAO,EAAE,EAAE,EAAC,OAAO,GAAE,CAAC;AAEjE,QAAM;WACE;AACR,MAAI,SAAS;AAGR,KAAG,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,qCAAqC,EAAE,CAAC;;;AAI/E,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACqD;AAGrD,QACE,CAAC,mBAAmB,KAAK,YAAY,EACrC,2CAA2C,cAC5C;CACD,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,aAAa,EAC/C,IAAI,OAA+B,cAAc,CAClD,CAAC,CACH;CACD,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,UAAU;EACpC,YAAY,OAAO,MAAM,GAAG,WAAW;EACxC;CAED,IAAI,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACpD,IAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,QACD,CACF;CACD,MAAM,aAAa,MAAM,GAAG,iBAAgB,QAC1C,IAAI,OAAO,YAAY,CAAC,UAAU,CACnC;CAED,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,EAAE;CACjC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;AACvB,KAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,GACvE;;CAIH,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,QAAO,CAAC;CAC5D,IAAI,MAAM;AAEV,YAAW,MAAM,QAAQ,YAAY;EACnC,MAAM,QAAQ;AACd,OAAK,MAAM,SAAS,OAAO,MAAM,MAAM,EAAE;AACvC,OAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,MAAM;AAEvD,OAAI,EAAE,QAAQ,SAAS,QAAQ;AAC7B,cAAU,KAAK,IAAI;AACnB,WAAO;AACP,UAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,QAAO,CAAC;AAC3C,UAAM;;;AAGV,mBAAiB,MAAM;AACvB,gBAAc,MAAM;AAEpB,MAAI,iBAAiB,qBAAqB;AACxC,SAAM;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;IAAO;AACvD;AACA,eAAY;AACZ,eAAY,EAAE;AACd,mBAAgB;;;AAKpB,KAAI,UAAU,SAAS,GAAG;AACxB,QAAM;GAAC,KAAK;GAAY,GAAG;GAAU;GAAW;GAAO;AACvD;AACA,cAAY;;AAGd,OAAM;EAAC,KAAK;EAAsB,GAAG;EAAU;EAAO;AACtD,YAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AAChD,IAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,MACf;;;;;;;;;;;AAYH,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CACF;CACD,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,KAAK;AACnD,KAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,SAAS;EAE/D,MAAM,EAAC,MAAM,aAAY,eAAe,SAAS;EACjD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;GAAK,CAAC,CAAC,IAAI,GAAG;AAClE,QAAM;AACN,QAAM,mBAAmB,OAAO,0BAA0B,SAAS,GAAG;EAEtE,MAAM,YAAY,qBAAqB,IAAI;AAC3C,KAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,GAAG;AACrE,SAAO;GAAC;GAAI;GAAU;UACf,GAAG;AAEV,QAAM,mBAAmB,OAEvB;8BACwB,SAAS,GAClC;AACD,KAAG,QAAQ,sCAAsC,EAAE;AACnD,QAAM;WACE;AACR,QAAM,mBAAmB,KAAK;;;AAIlC,SAAS,eACP,IACA,cACA,IACA,WAIC;AACD,QAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,aAAa;EAC5D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,KACnE;AACD,MAAI,CAAC,KACH,OAAM,IAAI,2BACR,IACA,oCACD;EAEH,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,oBAAoB;AACjE,MAAI,KAAK,cAAc,UAAU,UAC/B,OAAM,IAAI,2BACR,IACA,sDACD;AAEH,MAAI,KAAK,QAAQ,UAAU,YACzB,OAAM,IAAI,2BACR,IACA,oDACD;AAEH,MACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,OAAO,CAAC,EACtC,IAAI,IAAI,KAAK,uBAAuB,CACrC,CAED,OAAM,IAAI,2BACR,IACA,kDACD;EAEH,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,OAAO,EACnC,GAAG,OAAO,QAAQ,GAAG,QAAQ,CAC9B;AACD,OAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;AAC7B,OAAI,CAAC,QACH,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,8BACf;AAGH,OADgB,MAAQ,KAAK,qBAAqB,CACtC,WAAW,QAAQ,IAC7B,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,+CACf;;AAcL,SAAO;GAAC,WAAW;GAAM,UAXQ;IAC/B,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,OAAO,EAAC;KACjD;IACD,SAAS,OAAO,KAAK,GAAG,QAAQ,CAAC,QAC/B,QAAO,EAAE,OAAO,UAAU,QAC3B;IACD;IACD;GACiC;GAClC"}
1	+ {"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n DownloadStatus,\n JSONValue,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n makeBinarySelectExprs,\n makeDownloadStatements,\n type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n /*\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number \| undefined;\n\n /\n Use text-format COPY instead of binary COPY.\n * Binary is faster and handles all types (unknown types are cast to\n * `::text` in the SELECT). This flag exists as an escape hatch to\n * revert to the old code path if needed.\n /\n textCopy?: boolean \| undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b\|\\sLIMIT\\s+\\d/i;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n opts;\n const db = pgClient(lc, upstreamURI, 'backfill-stream', {\n ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n });\n let tx: TransactionPool \| undefined;\n let watermark: string;\n try {\n ({tx, watermark} = await createSnapshotTransaction(\n lc,\n upstreamURI,\n db,\n slot,\n ));\n const {tableSpec, backfill} = await validateSchema(\n tx,\n publications,\n bf,\n watermark,\n );\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n const stmts = makeDownloadStatements(tableSpec, cols);\n\n if (textCopy) {\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${stmts.select}) TO STDOUT`,\n new TsvParser(),\n cols.map(col => {\n const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n return (text: string) => parser(text) as JSONValue;\n }),\n flushThresholdBytes,\n );\n } else {\n const binaryStmts = makeDownloadStatements(\n tableSpec,\n cols,\n undefined,\n undefined,\n makeBinarySelectExprs(tableSpec, cols),\n );\n\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n new BinaryCopyParser(),\n cols.map(col => {\n const spec = tableSpec.columns[col];\n const decoder = hasBinaryDecoder(spec)\n ? makeBinaryDecoder(spec)\n : textCastDecoder;\n return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n }),\n flushThresholdBytes,\n );\n }\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx?.setDone();\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream<T>(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n {\n getTotalRows,\n getTotalBytes,\n }: Pick<DownloadStatements, 'getTotalRows' \| 'getTotalBytes'>,\n copyCommand: string,\n parser: {parse(chunk: Buffer): Iterable<T \| null>},\n decoders: ((field: T) => JSONValue)[],\n flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n // sync and must never appear in a backfill COPY.\n assert(\n !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n `backfill COPY must not sample or limit: ${copyCommand}`,\n );\n const start = performance.now();\n const [rows, bytes] = await tx.processReadTask(sql =>\n Promise.all([\n sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n ]),\n );\n const status: DownloadStatus = {\n rows: 0,\n totalRows: Number(rows[0].totalRows),\n totalBytes: Number(bytes[0].totalBytes),\n };\n\n let elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n {\n status,\n },\n );\n const copyStream = await tx.processReadTask(sql =>\n sql.unsafe(copyCommand).readable(),\n );\n\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: decoders.length});\n let col = 0;\n\n for await (const data of copyStream) {\n const chunk = data as Buffer;\n for (const field of parser.parse(chunk)) {\n row[col] = field === null ? null : decoders[col](field);\n\n if (++col === decoders.length) {\n rowValues.push(row);\n status.rows++;\n row = Array.from({length: decoders.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n }\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n }\n\n yield {tag: 'backfill-completed', ...backfill, status};\n elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n /\nasync function createSnapshotTransaction(\n lc: LogContext,\n upstreamURI: string,\n db: PostgresDB,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(\n lc,\n upstreamURI,\n 'backfill-replication-session',\n {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n },\n );\n const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, {slotName});\n\n const {init, imported} = importSnapshot(snapshot);\n const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n await imported;\n await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${slotName}\"`);\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return {tx, watermark};\n } catch (e) {\n // In the event of a failure, clean up the replication slot if created.\n await replicationSession.unsafe(\n /sql*/\n `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name = '${slotName}'`,\n );\n lc.warn?.(`Failed to create backfill snapshot`, e);\n throw e;\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n return {tableSpec: spec, backfill};\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAgEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,EAAE,EAC6B;AACrD,MAAK,GACF,YAAY,aAAa,WAAW,CACpC,YAAY,SAAS,GAAG,MAAM,KAAK;CAEtC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,SAAS,IAAI,aAAa,mBAAmB,GACrD,iBAAiB,MACnB,CAAC;CACF,IAAI;CACJ,IAAI;AACJ,KAAI;AACF,GAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,KACD;EACD,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,UACD;EAGD,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,QAAQ;EACrD,MAAM,QAAQ,uBAAuB,WAAW,KAAK;AAErD,MAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK,CAAC;AAClE,UAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,WAAW,EACf,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,QAAQ;AAClE,YAAQ,SAAiB,OAAO,KAAK;KACrC,EACF,oBACD;SACI;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,KAAK,CACvC;AAED,UAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,kBAAkB,EACtB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,KAAK,GAClC,kBAAkB,KAAK,GACvB;AACJ,YAAQ,QAAgB,QAAQ,IAAI;KACpC,EACF,oBACD;;UAEI,GAAG;AASV,MACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,qBAE7C,OAAM,IAAI,2BAA2B,IAAI,OAAO,EAAE,EAAE,EAAC,OAAO,GAAE,CAAC;AAEjE,QAAM;WACE;AACR,MAAI,SAAS;AAGR,KAAG,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,qCAAqC,EAAE,CAAC;;;AAI/E,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACqD;AAGrD,QACE,CAAC,mBAAmB,KAAK,YAAY,EACrC,2CAA2C,cAC5C;CACD,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,aAAa,EAC/C,IAAI,OAA+B,cAAc,CAClD,CAAC,CACH;CACD,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,UAAU;EACpC,YAAY,OAAO,MAAM,GAAG,WAAW;EACxC;CAED,IAAI,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACpD,IAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,QACD,CACF;CACD,MAAM,aAAa,MAAM,GAAG,iBAAgB,QAC1C,IAAI,OAAO,YAAY,CAAC,UAAU,CACnC;CAED,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,EAAE;CACjC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;AACvB,KAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,GACvE;;CAIH,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,QAAO,CAAC;CAC5D,IAAI,MAAM;AAEV,YAAW,MAAM,QAAQ,YAAY;EACnC,MAAM,QAAQ;AACd,OAAK,MAAM,SAAS,OAAO,MAAM,MAAM,EAAE;AACvC,OAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,MAAM;AAEvD,OAAI,EAAE,QAAQ,SAAS,QAAQ;AAC7B,cAAU,KAAK,IAAI;AACnB,WAAO;AACP,UAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,QAAO,CAAC;AAC3C,UAAM;;;AAGV,mBAAiB,MAAM;AACvB,gBAAc,MAAM;AAEpB,MAAI,iBAAiB,qBAAqB;AACxC,SAAM;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;IAAO;AACvD;AACA,eAAY;AACZ,eAAY,EAAE;AACd,mBAAgB;;;AAKpB,KAAI,UAAU,SAAS,GAAG;AACxB,QAAM;GAAC,KAAK;GAAY,GAAG;GAAU;GAAW;GAAO;AACvD;AACA,cAAY;;AAGd,OAAM;EAAC,KAAK;EAAsB,GAAG;EAAU;EAAO;AACtD,YAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AAChD,IAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,MACf;;;;;;;;;;;AAYH,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CACF;CACD,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,KAAK;AACnD,KAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,EAAC,UAAS,CAAC;EAEjE,MAAM,EAAC,MAAM,aAAY,eAAe,SAAS;EACjD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;GAAK,CAAC,CAAC,IAAI,GAAG;AAClE,QAAM;AACN,QAAM,mBAAmB,OAAO,0BAA0B,SAAS,GAAG;EAEtE,MAAM,YAAY,qBAAqB,IAAI;AAC3C,KAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,GAAG;AACrE,SAAO;GAAC;GAAI;GAAU;UACf,GAAG;AAEV,QAAM,mBAAmB,OAEvB;8BACwB,SAAS,GAClC;AACD,KAAG,OAAO,sCAAsC,EAAE;AAClD,QAAM;WACE;AACR,QAAM,mBAAmB,KAAK;;;AAIlC,SAAS,eACP,IACA,cACA,IACA,WAIC;AACD,QAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,aAAa;EAC5D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,KACnE;AACD,MAAI,CAAC,KACH,OAAM,IAAI,2BACR,IACA,oCACD;EAEH,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,oBAAoB;AACjE,MAAI,KAAK,cAAc,UAAU,UAC/B,OAAM,IAAI,2BACR,IACA,sDACD;AAEH,MAAI,KAAK,QAAQ,UAAU,YACzB,OAAM,IAAI,2BACR,IACA,oDACD;AAEH,MACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,OAAO,CAAC,EACtC,IAAI,IAAI,KAAK,uBAAuB,CACrC,CAED,OAAM,IAAI,2BACR,IACA,kDACD;EAEH,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,OAAO,EACnC,GAAG,OAAO,QAAQ,GAAG,QAAQ,CAC9B;AACD,OAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;AAC7B,OAAI,CAAC,QACH,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,8BACf;AAGH,OADgB,MAAQ,KAAK,qBAAqB,CACtC,WAAW,QAAQ,IAC7B,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,+CACf;;AAcL,SAAO;GAAC,WAAW;GAAM,UAXQ;IAC/B,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,OAAO,EAAC;KACjD;IACD,SAAS,OAAO,KAAK,GAAG,QAAQ,CAAC,QAC/B,QAAO,EAAE,OAAO,UAAU,QAC3B;IACD;IACD;GACiC;GAClC"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"~~AAKA~~,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;~~AAejD~~,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;~~AAOzD~~,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAGhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;~~AAuBjD~~;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,GACtB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAuC7E;~~AA4cD~~,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;~~AAqzBxD~~,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}
1	+ {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAGhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AAyBjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,GACtB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAuC7E;AA+ZD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAmzBxD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}

package/out/zero-cache/src/services/change-source/pg/change-source.js CHANGED Viewed

@@ -1,7 +1,6 @@
 import { assert } from "../../../../../shared/src/asserts.js";
 import { deepEqual } from "../../../../../shared/src/json.js";
 import { AbortError } from "../../../../../shared/src/abort-error.js";
-import { sleep } from "../../../../../shared/src/sleep.js";
 import { parse, valita_exports } from "../../../../../shared/src/valita.js";
 import { must } from "../../../../../shared/src/must.js";
 import { mapValues } from "../../../../../shared/src/objects.js";
@@ -15,24 +14,25 @@ import { StatementRunner } from "../../../db/statements.js";
 import { isPostgresError, pgClient } from "../../../types/pg.js";
 import { majorVersionFromString, majorVersionToString } from "../../../types/state-version.js";
 import { fromBigInt, toBigInt, toStateVersionString } from "./lsn.js";
-import { runTx } from "../../../db/run-transaction.js";
 import { getPublicationInfo } from "./schema/published.js";
 import { replicationEventSchema } from "./schema/ddl.js";
 import { validate } from "./schema/validation.js";
-import { dropShard, getInternalShardConfig, getReplicaAtVersion, internalPublicationPrefix, legacyReplicationSlot, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotExpression } from "./schema/shard.js";
+import { dropShard, getInternalShardConfig, getReplicaAtVersion, internalPublicationPrefix, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotPrefix } from "./schema/shard.js";
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import { initReplica } from "../common/replica-schema.js";
 import "../../../types/pg-versions.js";
 import { BackfillManager } from "../common/backfill-manager.js";
 import { ChangeStreamMultiplexer } from "../common/change-stream-multiplexer.js";
+import { dropOldReplicasAndSlots } from "./replication-slots.js";
 import { updateShardSchema } from "./schema/init.js";
 import { initialSync } from "./initial-sync.js";
 import { streamBackfill } from "./backfill-stream.js";
 import { subscribe } from "./logical-replication/stream.js";
 import { nanoid } from "nanoid";
 import postgres from "postgres";
-import { PG_ADMIN_SHUTDOWN, PG_INSUFFICIENT_PRIVILEGE, PG_OBJECT_IN_USE } from "@drdgvhbh/postgres-error-codes";
+import { PG_ADMIN_SHUTDOWN, PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/change-source.ts
+var REPLICA_SLOT_CLEANUP_INTERVAL_MS = 3e4;
 /**
 * Initializes a Postgres change source, including the initial sync of the
 * replica, before streaming changes from the corresponding logical replication
@@ -107,6 +107,7 @@ var PostgresChangeSource = class {
 	}
 	async stop() {
 		this.#lagReporter?.stop();
+		clearTimeout(this.#cleanupTimer);
 		await this.#db.end();
 	}
 	async startLagReporter() {
@@ -121,9 +122,9 @@ var PostgresChangeSource = class {
 		return null;
 	}
 	async startStream(clientWatermark, backfillRequests = []) {
-		const { slot } = this.#replica;
-		await this.#stopExistingReplicationSlotSubscribers(slot);
+		await this.#stopExistingReplicationSlotSubscriber();
 		const config = await getInternalShardConfig(this.#db, this.#shard);
+		const { slot } = this.#replica;
 		this.#lc.info?.(`starting replication stream@${slot}`);
 		return this.#startStream(slot, clientWatermark, config, backfillRequests);
 	}
@@ -214,73 +215,55 @@ var PostgresChangeSource = class {
 		}
 	}
 	/**
-	* Stops replication slots associated with this shard, and returns
-	* a `cleanup` task that drops any slot other than the specified
-	* `slotToKeep`.
-	*
-	* Note that replication slots created after `slotToKeep` (as indicated by
-	* the timestamp suffix) are preserved, as those are newly syncing replicas
-	* that will soon take over the slot.
+	* Stops replication slots associated with this shard, and asynchronously
+	* runs a cleanup task that drops older replicas and slots.
 	*/
-	async #stopExistingReplicationSlotSubscribers(slotToKeep) {
-		const slotExpression = replicationSlotExpression(this.#shard);
-		const legacySlotName = legacyReplicationSlot(this.#shard);
-		const result = await runTx(this.#db, async (sql) => {
-			const result = await sql`
-      SELECT slot_name as slot, pg_terminate_backend(active_pid) as terminated, active_pid as pid
-        FROM pg_replication_slots
-        WHERE (slot_name LIKE ${slotExpression} OR slot_name = ${legacySlotName})
-              AND slot_name <= ${slotToKeep}`;
-			this.#lc.info?.(`terminated replication slots: ${JSON.stringify(result)}`);
-			const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;
-			const replicasBefore = await sql`
-        SELECT slot, version, "initialSyncContext", "subscriberContext"
-          FROM ${sql(replicasTable)} ORDER BY slot`;
-			if (result.length === 0) {
-				const shardSlots = await sql`
+	async #stopExistingReplicationSlotSubscriber() {
+		const sql = this.#db;
+		const { id: replicaID, slot } = this.#replica;
+		const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;
+		const result = await sql`
+      SELECT pg_terminate_backend(active_pid) as terminated, active_pid as pid
+        FROM pg_replication_slots
+        WHERE slot_name = ${slot}
+    `;
+		if (result.length === 0) {
+			const slotExpression = replicationSlotPrefix(this.#shard);
+			const replicas = await sql`
+        SELECT id, rank, slot, version, "initialSyncContext", "subscriberContext"
+          FROM ${sql(replicasTable)} ORDER BY rank DESC`;
+			const slots = await sql`
         SELECT slot_name as slot, active, active_pid as pid
           FROM pg_replication_slots
-          WHERE slot_name LIKE ${slotExpression} OR slot_name = ${legacySlotName}
+          WHERE slot_name LIKE ${slotExpression}
           ORDER BY slot_name`;
-				this.#lc.warn?.(`slot ${slotToKeep} not found while cleaning subscribers`, {
-					slots: shardSlots,
-					replicas: replicasBefore
-				});
-				throw new AbortError(`replication slot ${slotToKeep} is missing. A different replication-manager should now be running on a new replication slot.`);
+			this.#lc.warn?.(`slot ${slot} not found while cleaning subscribers`, {
+				slots,
+				replicas
+			});
+			throw new AbortError(`replication slot ${slot} is missing. A different replication-manager should now be running on a new replication slot.`);
+		}
+		this.#lc.info?.(`terminated replication slots: ${JSON.stringify(result)}`);
+		await sql`
+      UPDATE ${sql(replicasTable)}
+        SET "subscriberContext" = ${this.#context}
+        WHERE id = ${replicaID}`;
+		this.#cleanUpOlderReplicasAndSlots();
+	}
+	#cleanupTimer;
+	async #cleanUpOlderReplicasAndSlots() {
+		clearTimeout(this.#cleanupTimer);
+		try {
+			const result = await dropOldReplicasAndSlots(this.#lc, this.#db, this.#shard, this.#replica.rank);
+			if (result.draining === 0) {
+				this.#lc.info?.(`finished cleaning up replicas and slots`, { result });
+				return;
 			}
-			this.#lc.info?.(`replicas before cleanup (slotToKeep=${slotToKeep}): ${JSON.stringify(replicasBefore)}`);
-			await sql`
-        DELETE FROM ${sql(replicasTable)} WHERE slot < ${slotToKeep}`;
-			await sql`
-        UPDATE ${sql(replicasTable)}
-          SET "subscriberContext" = ${this.#context}
-          WHERE slot = ${slotToKeep}`;
-			const replicasAfter = await sql`
-      SELECT slot, version FROM ${sql(replicasTable)} ORDER BY slot`;
-			this.#lc.info?.(`replicas after cleanup (slotToKeep=${slotToKeep}): ${JSON.stringify(replicasAfter)}`);
-			return result;
-		});
-		const pids = result.filter(({ pid }) => pid !== null).map(({ pid }) => pid);
-		if (pids.length) this.#lc.info?.(`signaled subscriber ${pids} to shut down`);
-		const otherSlots = result.filter(({ slot }) => slot !== slotToKeep).map(({ slot }) => slot);
-		if (otherSlots.length) this.#dropReplicationSlots(otherSlots).catch((e) => this.#lc.warn?.(`error dropping replication slots`, e));
-	}
-	async #dropReplicationSlots(slots) {
-		this.#lc.info?.(`dropping other replication slot(s) ${slots}`);
-		const sql = this.#db;
-		for (let i = 0; i < 5; i++) try {
-			await sql`
-          SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots
-            WHERE slot_name IN ${sql(slots)}
-        `;
-			this.#lc.info?.(`successfully dropped ${slots}`);
-			return;
+			this.#lc.info?.(`old slots still draining`, { result });
 		} catch (e) {
-			if (e instanceof postgres.PostgresError && e.code === PG_OBJECT_IN_USE) this.#lc.debug?.(`attempt ${i + 1}: ${String(e)}`, e);
-			else this.#lc.warn?.(`error dropping ${slots}`, e);
-			await sleep(1e3);
+			this.#lc.warn?.(`error dropping replication slots`, e);
 		}
-		this.#lc.warn?.(`maximum attempts exceeded dropping ${slots}`);
+		this.#cleanupTimer = setTimeout(() => this.#cleanUpOlderReplicasAndSlots(), REPLICA_SLOT_CLEANUP_INTERVAL_MS);
 	}
 };
 var Acker = class {