npm - @rocicorp/zero - Versions diffs - 1.9.1-canary.1 → 1.10.0-canary.1 - Mend

@rocicorp/zero 1.9.1-canary.1 → 1.10.0-canary.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (344) hide show

package/out/zero-cache/src/services/change-source/custom/change-source.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n AutoResetSignal,\n type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n createReplicationStateTables,\n getSubscriptionState,\n initReplicationState,\n type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n type BackfillRequest,\n type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\n/\n Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n /\nexport async function initializeCustomChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n context: ServerContext,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n (log, tx) => initialSync(log, shard, tx, upstreamURI, context),\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n if (shard.publications.length) {\n // Verify that the publications match what has been synced.\n const requested = shard.publications.toSorted();\n const replicated = subscriptionState.publications.sort();\n if (!deepEqual(requested, replicated)) {\n throw new Error(\n `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n );\n }\n }\n\n const changeSource = new CustomChangeSource(\n lc,\n upstreamURI,\n shard,\n subscriptionState,\n );\n\n return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replicationConfig: ReplicationConfig;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replicationConfig: ReplicationConfig,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replicationConfig = replicationConfig;\n }\n\n initialSync(): ChangeStream {\n return this.#startStream();\n }\n\n startLagReporter() {\n return null; // Not supported for custom sources\n }\n\n stop(): Promise<void> {\n return Promise.resolve();\n }\n\n startStream(\n clientWatermark: string,\n backfillRequests: BackfillRequest[] = [],\n ): Promise<ChangeStream> {\n if (backfillRequests?.length) {\n throw new Error(\n 'backfill is yet not supported for custom change sources',\n );\n }\n return Promise.resolve(this.#startStream(clientWatermark));\n }\n\n #startStream(clientWatermark?: string): ChangeStream {\n const {publications, replicaVersion} = this.#replicationConfig;\n const {appID, shardNum} = this.#shard;\n const url = new URL(this.#upstreamUri);\n url.searchParams.set('appID', appID);\n url.searchParams.set('shardNum', String(shardNum));\n for (const pub of publications) {\n url.searchParams.append('publications', pub);\n }\n if (clientWatermark) {\n assert(\n replicaVersion.length,\n 'replicaVersion is required when clientWatermark is set',\n );\n url.searchParams.set('lastWatermark', clientWatermark);\n url.searchParams.set('replicaVersion', replicaVersion);\n }\n\n const ws = new WebSocket(url);\n const {instream, outstream} = stream(\n this.#lc,\n ws,\n changeStreamMessageSchema,\n // Upstream acks coalesce. If upstream exhibits back-pressure,\n // only the last ACK is kept / buffered.\n {coalesce: (curr: ChangeSourceUpstream) => curr},\n );\n return {changes: instream, acks: outstream};\n }\n}\n\n/\n Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n \n Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n context: ServerContext,\n) {\n const {appID: id, publications} = shard;\n const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n replicaVersion: '', // ignored for initialSync()\n publications,\n });\n const {changes} = changeSource.initialSync();\n\n createReplicationStateTables(tx);\n const processor = new ChangeProcessor(\n new StatementRunner(tx),\n 'initial-sync',\n (_, err) => {\n throw err;\n },\n );\n\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n try {\n let num = 0;\n for await (const change of changes) {\n const [tag] = change;\n switch (tag) {\n case 'begin': {\n const {commitWatermark} = change[2];\n lc.info?.(\n `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying upstream tables at version ${commitWatermark}`,\n 5000,\n );\n initReplicationState(\n tx,\n publications.toSorted(),\n commitWatermark,\n context,\n false,\n );\n processor.processMessage(lc, change);\n break;\n }\n case 'data':\n processor.processMessage(lc, change);\n if (++num % 1000 === 0) {\n lc.debug?.(`processed ${num} changes`);\n }\n break;\n case 'commit':\n processor.processMessage(lc, change);\n validateInitiallySyncedData(lc, tx, shard);\n lc.info?.(`finished initial-sync of ${num} changes`);\n return;\n\n case 'status':\n break; // Ignored\n // @ts-expect-error: falls through if the tag is not 'reset-required\n case 'control': {\n const {tag, message} = change[1];\n if (tag === 'reset-required') {\n throw new AutoResetSignal(\n message ?? 'auto-reset signaled by change source',\n );\n }\n }\n // falls through\n case 'rollback':\n throw new Error(\n `unexpected message during initial-sync: ${stringify(change)}`,\n );\n default:\n unreachable(change);\n }\n }\n throw new Error(\n `change source ${upstreamURI} closed before initial-sync completed`,\n );\n } catch (e) {\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n appID,\n shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n return {\n [`${appID}_${shardNum}.clients`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n lastMutationID: {type: 'number'},\n userID: {type: 'string'},\n },\n [`${appID}_${shardNum}.mutations`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n mutationID: {type: 'number'},\n mutation: {type: 'json'},\n },\n [`${appID}.permissions`]: {\n permissions: {type: 'json'},\n hash: {type: 'string'},\n },\n };\n}\n\nfunction validateInitiallySyncedData(\n lc: LogContext,\n db: Database,\n shard: ShardID,\n) {\n const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n const required = getRequiredTables(shard);\n for (const [name, columns] of Object.entries(required)) {\n const table = tables.get(name)?.zqlSpec;\n if (!table) {\n throw new Error(\n `Upstream is missing the \"${name}\" table. (Found ${[\n ...tables.keys(),\n ]})` +\n `Please ensure that each table has a unique index over one ` +\n `or more non-null columns.`,\n );\n }\n for (const [col, {type}] of Object.entries(columns)) {\n const found = table[col];\n if (!found) {\n throw new Error(\n `Upstream \"${table}\" table is missing the \"${col}\" column`,\n );\n }\n if (found.type !== type) {\n throw new Error(\n `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SAC6E;CAC7E,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,OAAO,CAC/D;CAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;CAC9C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;CAC3E,QAAQ,MAAM;CAEd,IAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,SAAS;EAC9C,MAAM,aAAa,kBAAkB,aAAa,KAAK;EACvD,IAAI,CAAC,UAAU,WAAW,UAAU,GAClC,MAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,EAC9G;CAEJ;CASA,OAAO;EAAC;EAAmB,cAAA,IAPF,mBACvB,IACA,aACA,OACA,iBAGyB;CAAY;AACzC;AAEA,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,eAAe;EACpB,KAAKC,SAAS;EACd,KAAKC,qBAAqB;CAC5B;CAEA,cAA4B;EAC1B,OAAO,KAAKC,aAAa;CAC3B;CAEA,mBAAmB;EACjB,OAAO;CACT;CAEA,OAAsB;EACpB,OAAO,QAAQ,QAAQ;CACzB;CAEA,YACE,iBACA,mBAAsC,CAAC,GAChB;EACvB,IAAI,kBAAkB,QACpB,MAAM,IAAI,MACR,yDACF;EAEF,OAAO,QAAQ,QAAQ,KAAKA,aAAa,eAAe,CAAC;CAC3D;CAEA,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,KAAKD;EAC5C,MAAM,EAAC,OAAO,aAAY,KAAKD;EAC/B,MAAM,MAAM,IAAI,IAAI,KAAKD,YAAY;EACrC,IAAI,aAAa,IAAI,SAAS,KAAK;EACnC,IAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;EACjD,KAAK,MAAM,OAAO,cAChB,IAAI,aAAa,OAAO,gBAAgB,GAAG;EAE7C,IAAI,iBAAiB;GACnB,OACE,eAAe,QACf,wDACF;GACA,IAAI,aAAa,IAAI,iBAAiB,eAAe;GACrD,IAAI,aAAa,IAAI,kBAAkB,cAAc;EACvD;EAEA,MAAM,KAAK,IAAI,YAAU,GAAG;EAC5B,MAAM,EAAC,UAAU,cAAa,OAC5B,KAAKD,KACL,IACA,2BAGA,EAAC,WAAW,SAA+B,KAAI,CACjD;EACA,OAAO;GAAC,SAAS;GAAU,MAAM;EAAS;CAC5C;AACF;;;;;;;;;;;;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAAW,IAJO,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;CACF,CACkB,EAAa,YAAY;CAE3C,6BAA6B,EAAE;CAC/B,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,EAAE,GACtB,iBACC,GAAG,QAAQ;EACV,MAAM;CACR,CACF;CAEA,MAAM,kBAAkB,2BAA2B,sBAAsB,EAAE;CAC3E,IAAI;EACF,IAAI,MAAM;EACV,WAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;GACd,QAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;KACjC,GAAG,OACD,yBAAyB,GAAG,qBAAqB,iBACnD;KACA,gBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,GACF;KACA,qBACE,IACA,aAAa,SAAS,GACtB,iBACA,SACA,KACF;KACA,UAAU,eAAe,IAAI,MAAM;KACnC;IACF;IACA,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,IAAI,EAAE,MAAM,QAAS,GACnB,GAAG,QAAQ,aAAa,IAAI,SAAS;KAEvC;IACF,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,4BAA4B,IAAI,IAAI,KAAK;KACzC,GAAG,OAAO,4BAA4B,IAAI,SAAS;KACnD;IAEF,KAAK,UACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;KAC9B,IAAI,QAAQ,kBACV,MAAM,IAAI,gBACR,WAAW,sCACb;IAEJ;IAEA,KAAK,YACH,MAAM,IAAI,MACR,2CAA2C,UAAU,MAAM,GAC7D;IACF,SACE,YAAY,MAAM;GACtB;EACF;EACA,MAAM,IAAI,MACR,iBAAiB,YAAY,sCAC/B;CACF,SAAS,GAAG;EACV,MAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;CAClE,UAAU;EACR,gBAAgB,KAAK;CACvB;AACF;AAIA,SAAS,kBAAkB,EACzB,OACA,YACuD;CACvD,OAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,gBAAgB,EAAC,MAAM,SAAQ;GAC/B,QAAQ,EAAC,MAAM,SAAQ;EACzB;GACC,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,YAAY,EAAC,MAAM,SAAQ;GAC3B,UAAU,EAAC,MAAM,OAAM;EACzB;GACC,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,OAAM;GAC1B,MAAM,EAAC,MAAM,SAAQ;EACvB;CACF;AACF;AAEA,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,KAAI,CAAC;CACxE,MAAM,WAAW,kBAAkB,KAAK;CACxC,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,GAAG;EACtD,MAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;EAChC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,KAAK,CACjB,EAAE,qFAGJ;EAEF,KAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,OAAO,GAAG;GACnD,MAAM,QAAQ,MAAM;GACpB,IAAI,CAAC,OACH,MAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,SACnD;GAEF,IAAI,MAAM,SAAS,MACjB,MAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,OAClF;EAEJ;CACF;AACF"}
1	+ {"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n AutoResetSignal,\n type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n createReplicationStateTables,\n getSubscriptionState,\n initReplicationState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {\n restoreReplica,\n type InitializeResult,\n type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n type BackfillRequest,\n type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\n/\n Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n /\nexport async function initializeCustomChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n context: ServerContext,\n {litestream, constraints}: RestoreOptions = {},\n): Promise<InitializeResult> {\n // At the moment, the custom change-source implementation does not support\n // per-RM backups (and thus does not support litestream v5 or RMv2). The\n // current proposal to support this without requiring additional upstream\n // state tracking is to:\n // - use litestream to (ltx) query a fixed pool of subfolders within the\n // backupURL (e.g. /a/, /b/, /c/, /d/, /e/)\n // - restore from the subfolder with the latest ltx file\n // - backup to the subfolder with the oldest (or non-existent) ltx file\n if (litestream?.backupURL) {\n await restoreReplica(lc, litestream, replicaDbFile, constraints);\n }\n\n let initialSynced = false;\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n (log, tx) => {\n initialSynced = true;\n return initialSync(log, shard, tx, upstreamURI, context);\n },\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n if (shard.publications.length) {\n // Verify that the publications match what has been synced.\n const requested = shard.publications.toSorted();\n const replicated = subscriptionState.publications.sort();\n if (!deepEqual(requested, replicated)) {\n throw new Error(\n `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n );\n }\n }\n\n const changeSource = new CustomChangeSource(\n lc,\n upstreamURI,\n shard,\n subscriptionState,\n );\n\n return {\n subscriptionState,\n changeSource,\n destinationBackupURL: litestream?.backupURL,\n // A custom change source has no upstream `replicas` table to identify\n // itself from, so the generation is the whole of the SQLite change log's\n // identity here.\n replicaID: null,\n waitForBackupBeforeServing: initialSynced,\n };\n}\n\nclass CustomChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replicationConfig: ReplicationConfig;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replicationConfig: ReplicationConfig,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replicationConfig = replicationConfig;\n }\n\n initialSync(): ChangeStream {\n return this.#startStream();\n }\n\n startLagReporter() {\n return null; // Not supported for custom sources\n }\n\n stop(): Promise<void> {\n return Promise.resolve();\n }\n\n startStream(\n clientWatermark: string,\n backfillRequests: BackfillRequest[] = [],\n ): Promise<ChangeStream> {\n if (backfillRequests?.length) {\n throw new Error(\n 'backfill is yet not supported for custom change sources',\n );\n }\n return Promise.resolve(this.#startStream(clientWatermark));\n }\n\n #startStream(clientWatermark?: string): ChangeStream {\n const {publications, replicaVersion} = this.#replicationConfig;\n const {appID, shardNum} = this.#shard;\n const url = new URL(this.#upstreamUri);\n url.searchParams.set('appID', appID);\n url.searchParams.set('shardNum', String(shardNum));\n for (const pub of publications) {\n url.searchParams.append('publications', pub);\n }\n if (clientWatermark) {\n assert(\n replicaVersion.length,\n 'replicaVersion is required when clientWatermark is set',\n );\n url.searchParams.set('lastWatermark', clientWatermark);\n url.searchParams.set('replicaVersion', replicaVersion);\n }\n\n const ws = new WebSocket(url);\n const {instream, outstream} = stream(\n this.#lc,\n ws,\n changeStreamMessageSchema,\n // Upstream acks coalesce. If upstream exhibits back-pressure,\n // only the last ACK is kept / buffered.\n {coalesce: (curr: ChangeSourceUpstream) => curr},\n );\n return {changes: instream, acks: outstream};\n }\n}\n\n/\n Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n \n Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n context: ServerContext,\n) {\n const {appID: id, publications} = shard;\n const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n replicaVersion: '', // ignored for initialSync()\n publications,\n });\n const {changes} = changeSource.initialSync();\n\n createReplicationStateTables(tx);\n const processor = new ChangeProcessor(\n new StatementRunner(tx),\n 'initial-sync',\n (_, err) => {\n throw err;\n },\n );\n\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n try {\n let num = 0;\n for await (const change of changes) {\n const [tag] = change;\n switch (tag) {\n case 'begin': {\n const {commitWatermark} = change[2];\n lc.info?.(\n `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying upstream tables at version ${commitWatermark}`,\n 5000,\n );\n initReplicationState(\n tx,\n publications.toSorted(),\n commitWatermark,\n context,\n false,\n );\n processor.processMessage(lc, change);\n break;\n }\n case 'data':\n processor.processMessage(lc, change);\n if (++num % 1000 === 0) {\n lc.debug?.(`processed ${num} changes`);\n }\n break;\n case 'commit':\n processor.processMessage(lc, change);\n validateInitiallySyncedData(lc, tx, shard);\n lc.info?.(`finished initial-sync of ${num} changes`);\n return;\n\n case 'status':\n break; // Ignored\n // @ts-expect-error: falls through if the tag is not 'reset-required\n case 'control': {\n const {tag, message} = change[1];\n if (tag === 'reset-required') {\n throw new AutoResetSignal(\n message ?? 'auto-reset signaled by change source',\n );\n }\n }\n // falls through\n case 'rollback':\n throw new Error(\n `unexpected message during initial-sync: ${stringify(change)}`,\n );\n default:\n unreachable(change);\n }\n }\n throw new Error(\n `change source ${upstreamURI} closed before initial-sync completed`,\n );\n } catch (e) {\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n appID,\n shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n return {\n [`${appID}_${shardNum}.clients`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n lastMutationID: {type: 'number'},\n userID: {type: 'string'},\n },\n [`${appID}_${shardNum}.mutations`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n mutationID: {type: 'number'},\n mutation: {type: 'json'},\n },\n [`${appID}.permissions`]: {\n permissions: {type: 'json'},\n hash: {type: 'string'},\n },\n };\n}\n\nfunction validateInitiallySyncedData(\n lc: LogContext,\n db: Database,\n shard: ShardID,\n) {\n const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n const required = getRequiredTables(shard);\n for (const [name, columns] of Object.entries(required)) {\n const table = tables.get(name)?.zqlSpec;\n if (!table) {\n throw new Error(\n `Upstream is missing the \"${name}\" table. (Found ${[\n ...tables.keys(),\n ]})` +\n `Please ensure that each table has a unique index over one ` +\n `or more non-null columns.`,\n );\n }\n for (const [col, {type}] of Object.entries(columns)) {\n const found = table[col];\n if (!found) {\n throw new Error(\n `Upstream \"${table}\" table is missing the \"${col}\" column`,\n );\n }\n if (found.type !== type) {\n throw new Error(\n `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA6CA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SACA,EAAC,YAAY,gBAA+B,CAAC,GAClB;CAS3B,IAAI,YAAY,WACd,MAAM,eAAe,IAAI,YAAY,eAAe,WAAW;CAGjE,IAAI,gBAAgB;CACpB,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO;EACX,gBAAgB;EAChB,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,OAAO;CACzD,CACF;CAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;CAC9C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;CAC3E,QAAQ,MAAM;CAEd,IAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,SAAS;EAC9C,MAAM,aAAa,kBAAkB,aAAa,KAAK;EACvD,IAAI,CAAC,UAAU,WAAW,UAAU,GAClC,MAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,EAC9G;CAEJ;CASA,OAAO;EACL;EACA,cAAA,IATuB,mBACvB,IACA,aACA,OACA,iBAKA;EACA,sBAAsB,YAAY;EAIlC,WAAW;EACX,4BAA4B;CAC9B;AACF;AAEA,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,eAAe;EACpB,KAAKC,SAAS;EACd,KAAKC,qBAAqB;CAC5B;CAEA,cAA4B;EAC1B,OAAO,KAAKC,aAAa;CAC3B;CAEA,mBAAmB;EACjB,OAAO;CACT;CAEA,OAAsB;EACpB,OAAO,QAAQ,QAAQ;CACzB;CAEA,YACE,iBACA,mBAAsC,CAAC,GAChB;EACvB,IAAI,kBAAkB,QACpB,MAAM,IAAI,MACR,yDACF;EAEF,OAAO,QAAQ,QAAQ,KAAKA,aAAa,eAAe,CAAC;CAC3D;CAEA,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,KAAKD;EAC5C,MAAM,EAAC,OAAO,aAAY,KAAKD;EAC/B,MAAM,MAAM,IAAI,IAAI,KAAKD,YAAY;EACrC,IAAI,aAAa,IAAI,SAAS,KAAK;EACnC,IAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;EACjD,KAAK,MAAM,OAAO,cAChB,IAAI,aAAa,OAAO,gBAAgB,GAAG;EAE7C,IAAI,iBAAiB;GACnB,OACE,eAAe,QACf,wDACF;GACA,IAAI,aAAa,IAAI,iBAAiB,eAAe;GACrD,IAAI,aAAa,IAAI,kBAAkB,cAAc;EACvD;EAEA,MAAM,KAAK,IAAI,UAAU,GAAG;EAC5B,MAAM,EAAC,UAAU,cAAa,OAC5B,KAAKD,KACL,IACA,2BAGA,EAAC,WAAW,SAA+B,KAAI,CACjD;EACA,OAAO;GAAC,SAAS;GAAU,MAAM;EAAS;CAC5C;AACF;;;;;;;;;;;;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAAW,IAJO,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;CACF,CACkB,EAAa,YAAY;CAE3C,6BAA6B,EAAE;CAC/B,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,EAAE,GACtB,iBACC,GAAG,QAAQ;EACV,MAAM;CACR,CACF;CAEA,MAAM,kBAAkB,2BAA2B,sBAAsB,EAAE;CAC3E,IAAI;EACF,IAAI,MAAM;EACV,WAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;GACd,QAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;KACjC,GAAG,OACD,yBAAyB,GAAG,qBAAqB,iBACnD;KACA,gBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,GACF;KACA,qBACE,IACA,aAAa,SAAS,GACtB,iBACA,SACA,KACF;KACA,UAAU,eAAe,IAAI,MAAM;KACnC;IACF;IACA,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,IAAI,EAAE,MAAM,QAAS,GACnB,GAAG,QAAQ,aAAa,IAAI,SAAS;KAEvC;IACF,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,4BAA4B,IAAI,IAAI,KAAK;KACzC,GAAG,OAAO,4BAA4B,IAAI,SAAS;KACnD;IAEF,KAAK,UACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;KAC9B,IAAI,QAAQ,kBACV,MAAM,IAAI,gBACR,WAAW,sCACb;IAEJ;IAEA,KAAK,YACH,MAAM,IAAI,MACR,2CAA2C,UAAU,MAAM,GAC7D;IACF,SACE,YAAY,MAAM;GACtB;EACF;EACA,MAAM,IAAI,MACR,iBAAiB,YAAY,sCAC/B;CACF,SAAS,GAAG;EACV,MAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;CAClE,UAAU;EACR,gBAAgB,KAAK;CACvB;AACF;AAIA,SAAS,kBAAkB,EACzB,OACA,YACuD;CACvD,OAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,gBAAgB,EAAC,MAAM,SAAQ;GAC/B,QAAQ,EAAC,MAAM,SAAQ;EACzB;GACC,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,YAAY,EAAC,MAAM,SAAQ;GAC3B,UAAU,EAAC,MAAM,OAAM;EACzB;GACC,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,OAAM;GAC1B,MAAM,EAAC,MAAM,SAAQ;EACvB;CACF;AACF;AAEA,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,KAAI,CAAC;CACxE,MAAM,WAAW,kBAAkB,KAAK;CACxC,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,GAAG;EACtD,MAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;EAChC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,KAAK,CACjB,EAAE,qFAGJ;EAEF,KAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,OAAO,GAAG;GACnD,MAAM,QAAQ,MAAM;GACpB,IAAI,CAAC,OACH,MAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,SACnD;GAEF,IAAI,MAAM,SAAS,MACjB,MAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,OAClF;EAEJ;CACF;AACF"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js CHANGED Viewed

@@ -139,25 +139,22 @@ async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
 	});
 	const slotName = `${slotNamePrefix}_bf_${Date.now()}`;
 	try {
-		const { snapshot_name: snapshot, consistent_point: lsn } = await createReplicationSlot(lc, replicationSession, { slotName });
+		const { snapshot_name: snapshot, consistent_point: lsn } = await createReplicationSlot(lc, replicationSession, {
+			slotName,
+			temporary: true
+		});
 		const { init, imported } = importSnapshot(snapshot);
 		const tx = new TransactionPool(lc, {
 			mode: READONLY,
 			init
 		}).run(db);
 		await imported;
-		await replicationSession.unsafe(`DROP_REPLICATION_SLOT "${slotName}"`);
 		const watermark = toStateVersionString(lsn);
 		lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);
 		return {
 			tx,
 			watermark
 		};
-	} catch (e) {
-		await replicationSession.unsafe(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots
-         WHERE slot_name = '${slotName}'`);
-		lc.warn?.(`Failed to create backfill snapshot`, e);
-		throw e;
 	} finally {
 		await replicationSession.end();
 	}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n DownloadStatus,\n JSONValue,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n makeBinarySelectExprs,\n makeDownloadStatements,\n type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n /*\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number \| undefined;\n\n /\n Use text-format COPY instead of binary COPY.\n * Binary is faster and handles all types (unknown types are cast to\n * `::text` in the SELECT). This flag exists as an escape hatch to\n * revert to the old code path if needed.\n /\n textCopy?: boolean \| undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b\|\\sLIMIT\\s+\\d/i;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n opts;\n const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n });\n let tx: TransactionPool \| undefined;\n let watermark: string;\n try {\n ({tx, watermark} = await createSnapshotTransaction(\n lc,\n upstreamURI,\n db,\n slot,\n ));\n const {tableSpec, backfill} = await validateSchema(\n tx,\n publications,\n bf,\n watermark,\n );\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n const stmts = makeDownloadStatements(tableSpec, cols);\n\n if (textCopy) {\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${stmts.select}) TO STDOUT`,\n new TsvParser(),\n cols.map(col => {\n const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n return (text: string) => parser(text) as JSONValue;\n }),\n flushThresholdBytes,\n );\n } else {\n const binaryStmts = makeDownloadStatements(\n tableSpec,\n cols,\n undefined,\n undefined,\n makeBinarySelectExprs(tableSpec, cols),\n );\n\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n new BinaryCopyParser(),\n cols.map(col => {\n const spec = tableSpec.columns[col];\n const decoder = hasBinaryDecoder(spec)\n ? makeBinaryDecoder(spec)\n : textCastDecoder;\n return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n }),\n flushThresholdBytes,\n );\n }\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx?.setDone();\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream<T>(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n {\n getTotalRows,\n getTotalBytes,\n }: Pick<DownloadStatements, 'getTotalRows' \| 'getTotalBytes'>,\n copyCommand: string,\n parser: {parse(chunk: Buffer): Iterable<T \| null>},\n decoders: ((field: T) => JSONValue)[],\n flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n // sync and must never appear in a backfill COPY.\n assert(\n !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n `backfill COPY must not sample or limit: ${copyCommand}`,\n );\n const start = performance.now();\n const [rows, bytes] = await tx.processReadTask(sql =>\n Promise.all([\n sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n ]),\n );\n const status: DownloadStatus = {\n rows: 0,\n totalRows: Number(rows[0].totalRows),\n totalBytes: Number(bytes[0].totalBytes),\n };\n\n let elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n {\n status,\n },\n );\n const copyStream = await tx.processReadTask(sql =>\n sql.unsafe(copyCommand).readable(),\n );\n\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: decoders.length});\n let col = 0;\n\n for await (const data of copyStream) {\n const chunk = data as Buffer;\n for (const field of parser.parse(chunk)) {\n row[col] = field === null ? null : decoders[col](field);\n\n if (++col === decoders.length) {\n rowValues.push(row);\n status.rows++;\n row = Array.from({length: decoders.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n }\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n }\n\n yield {tag: 'backfill-completed', ...backfill, status};\n elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n /\nasync function createSnapshotTransaction(\n lc: LogContext,\n upstreamURI: string,\n db: PostgresDB,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(\n lc,\n upstreamURI,\n 'backfill-replication-session',\n {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n },\n );\n const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, {slotName});\n\n const {init, imported} = importSnapshot(snapshot);\n const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n await imported;\n await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${slotName}\"`);\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return {tx, watermark};\n } catch (e) {\n // In the event of a failure, clean up the replication slot if created.\n await replicationSession.unsafe(\n /sql*/\n `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name = '${slotName}'`,\n );\n lc.warn?.(`Failed to create backfill snapshot`, e);\n throw e;\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n return {tableSpec: spec, backfill};\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAgEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GAC8B;CACrD,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB,GAClE,iBAAiB,KACpB,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EACpD,MAAM,QAAQ,uBAAuB,WAAW,IAAI;EAEpD,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,GACD,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,IAAI,CACvC;GAEA,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,GACD,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACqD;CAGrD,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CACA,MAAM,aAAa,MAAM,GAAG,iBAAgB,QAC1C,IAAI,OAAO,WAAW,EAAE,SAAS,CACnC;CAEA,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAGA,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,WAAW,MAAM,QAAQ,YAAY;EACnC,MAAM,QAAQ;EACd,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;GAEtD,IAAI,EAAE,QAAQ,SAAS,QAAQ;IAC7B,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,qBAAqB;GACxC,MAAM;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;GAAM;GACtD;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;CACF;CAGA,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GAAC,KAAK;GAAY,GAAG;GAAU;GAAW;EAAM;EACtD;EACA,WAAW;CACb;CAEA,MAAM;EAAC,KAAK;EAAsB,GAAG;EAAU;CAAM;CACrD,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,EAAC,SAAQ,CAAC;EAEhE,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM;EACN,MAAM,mBAAmB,OAAO,0BAA0B,SAAS,EAAE;EAErE,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,SAAS,GAAG;EAEV,MAAM,mBAAmB,OAEvB;8BACwB,SAAS,EACnC;EACA,GAAG,OAAO,sCAAsC,CAAC;EACjD,MAAM;CACR,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,CAC9B;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAYA,OAAO;GAAC,WAAW;GAAM,UAAA;IAVvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAEuB;EAAQ;CACnC,CAAC;AACH"}
1	+ {"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n DownloadStatus,\n JSONValue,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n makeBinarySelectExprs,\n makeDownloadStatements,\n type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n /*\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number \| undefined;\n\n /\n Use text-format COPY instead of binary COPY.\n * Binary is faster and handles all types (unknown types are cast to\n * `::text` in the SELECT). This flag exists as an escape hatch to\n * revert to the old code path if needed.\n /\n textCopy?: boolean \| undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b\|\\sLIMIT\\s+\\d/i;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n opts;\n const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n });\n let tx: TransactionPool \| undefined;\n let watermark: string;\n try {\n ({tx, watermark} = await createSnapshotTransaction(\n lc,\n upstreamURI,\n db,\n slot,\n ));\n const {tableSpec, backfill} = await validateSchema(\n tx,\n publications,\n bf,\n watermark,\n );\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n const stmts = makeDownloadStatements(tableSpec, cols);\n\n if (textCopy) {\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${stmts.select}) TO STDOUT`,\n new TsvParser(),\n cols.map(col => {\n const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n return (text: string) => parser(text) as JSONValue;\n }),\n flushThresholdBytes,\n );\n } else {\n const binaryStmts = makeDownloadStatements(\n tableSpec,\n cols,\n undefined,\n undefined,\n makeBinarySelectExprs(tableSpec, cols),\n );\n\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n new BinaryCopyParser(),\n cols.map(col => {\n const spec = tableSpec.columns[col];\n const decoder = hasBinaryDecoder(spec)\n ? makeBinaryDecoder(spec)\n : textCastDecoder;\n return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n }),\n flushThresholdBytes,\n );\n }\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx?.setDone();\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream<T>(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n {\n getTotalRows,\n getTotalBytes,\n }: Pick<DownloadStatements, 'getTotalRows' \| 'getTotalBytes'>,\n copyCommand: string,\n parser: {parse(chunk: Buffer): Iterable<T \| null>},\n decoders: ((field: T) => JSONValue)[],\n flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n // sync and must never appear in a backfill COPY.\n assert(\n !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n `backfill COPY must not sample or limit: ${copyCommand}`,\n );\n const start = performance.now();\n const [rows, bytes] = await tx.processReadTask(sql =>\n Promise.all([\n sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n ]),\n );\n const status: DownloadStatus = {\n rows: 0,\n totalRows: Number(rows[0].totalRows),\n totalBytes: Number(bytes[0].totalBytes),\n };\n\n let elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n {\n status,\n },\n );\n const copyStream = await tx.processReadTask(sql =>\n sql.unsafe(copyCommand).readable(),\n );\n\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: decoders.length});\n let col = 0;\n\n for await (const data of copyStream) {\n const chunk = data as Buffer;\n for (const field of parser.parse(chunk)) {\n row[col] = field === null ? null : decoders[col](field);\n\n if (++col === decoders.length) {\n rowValues.push(row);\n status.rows++;\n row = Array.from({length: decoders.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n }\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n }\n\n yield {tag: 'backfill-completed', ...backfill, status};\n elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n */\nasync function createSnapshotTransaction(\n lc: LogContext,\n upstreamURI: string,\n db: PostgresDB,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(\n lc,\n upstreamURI,\n 'backfill-replication-session',\n {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n },\n );\n const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, {\n slotName,\n temporary: true, // deletes the slot when the replicationSession ends\n });\n\n const {init, imported} = importSnapshot(snapshot);\n const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n await imported;\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return {tx, watermark};\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n return {tableSpec: spec, backfill};\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAgEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GAC8B;CACrD,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB,GAClE,iBAAiB,KACpB,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EACpD,MAAM,QAAQ,uBAAuB,WAAW,IAAI;EAEpD,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,GACD,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,IAAI,CACvC;GAEA,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,GACD,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACqD;CAGrD,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CACA,MAAM,aAAa,MAAM,GAAG,iBAAgB,QAC1C,IAAI,OAAO,WAAW,EAAE,SAAS,CACnC;CAEA,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAGA,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,WAAW,MAAM,QAAQ,YAAY;EACnC,MAAM,QAAQ;EACd,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;GAEtD,IAAI,EAAE,QAAQ,SAAS,QAAQ;IAC7B,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,qBAAqB;GACxC,MAAM;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;GAAM;GACtD;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;CACF;CAGA,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GAAC,KAAK;GAAY,GAAG;GAAU;GAAW;EAAM;EACtD;EACA,WAAW;CACb;CAEA,MAAM;EAAC,KAAK;EAAsB,GAAG;EAAU;CAAM;CACrD,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;EACb,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM;EAEN,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,CAC9B;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAYA,OAAO;GAAC,WAAW;GAAM,UAAA;IAVvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAEuB;EAAQ;CACnC,CAAC;AACH"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts CHANGED Viewed

@@ -5,22 +5,21 @@ import { type LexiVersion } from '../../../types/lexi-version.ts';
 import { type PostgresDB } from '../../../types/pg.ts';
 import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
 import type { Sink } from '../../../types/streams.ts';
-import { type SubscriptionState } from '../../replicator/schema/replication-state.ts';
-import type { ChangeSource } from '../change-source.ts';
 import { type Listener } from '../common/change-stream-multiplexer.ts';
+import { type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
 import type { DownstreamStatusMessage } from '../protocol/current.ts';
 import type { ChangeStreamMessage } from '../protocol/current/downstream.ts';
-import { type InitialSyncOptions, type ServerContext } from './initial-sync.ts';
+import { type InitialSyncOptions, type ReplicaOptions, type ServerContext } from './initial-sync.ts';
 import type { MessageMessage, MessageRelation as PostgresRelation } from './logical-replication/pgoutput.types.ts';
+interface PurgeLock {
+    release(): Promise<void>;
+}
 /**
  * Initializes a Postgres change source, including the initial sync of the
  * replica, before streaming changes from the corresponding logical replication
  * stream.
  */
-export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, streamInboundTimeoutMs?: number | undefined): Promise<{
-    subscriptionState: SubscriptionState;
-    changeSource: ChangeSource;
-}>;
+export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, restoreOptions?: RestoreOptions, { backupV5 }?: ReplicaOptions, purgeLock?: PurgeLock | null, streamInboundTimeoutMs?: number | undefined): Promise<InitializeResult>;
 export declare class Acker implements Listener {
     #private;
     constructor(acks: Sink<bigint>);
@@ -40,6 +39,7 @@ export declare class LagReporter {
     constructor(lc: LogContext, shard: ShardID, db: PostgresDB, lagIntervalMs: number);
     get pgVersion(): number | undefined;
     initiateLagReport(log?: boolean): Promise<{
+        firstCommitTimeMs: number;
         nextSendTimeMs: number;
     }>;
     /**

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,OAAO,EACb,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;~~AAEpD~~,OAAO,EAEL,KAAK,~~iBAAiB~~,~~EAEvB~~,MAAM,~~8CAA8C~~,CAAC;~~AACtD~~,OAAO,KAAK,~~EAAC~~,~~YAAY~~,~~EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,~~KAAK,~~QAAQ~~,~~EACd~~,MAAM,~~wCAAwC~~,CAAC;~~AAEhD~~,OAAO,KAAK,EAEV,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;~~AA0BjD~~;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,~~sBAAsB~~,CAAC,EAAE,~~MAAM~~,GAAG,~~SAAS~~,~~GAC1C~~,~~OAAO~~,CAAC~~;IAAC~~,~~iBAAiB,~~EAAE,~~iBAAiB~~,~~CAAC;IAAC~~,~~YAAY~~,~~EAAE~~,~~YAAY~~,~~CAAA;~~CAAC,CAAC,~~CAwC7E~~;~~AA6aD~~,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;gBAqB7B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM;IAiBvB,IAAI,SAAS,uBAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ~~;;;~~IA0EnC;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS;IAcjE,IAAI;IA2CJ,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB;CAiDtE;AAunBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}
1	+ {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,OAAO,EACb,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAQpD,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,KAAK,EAEV,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AA6BjD,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EAAC,QAAQ,EAAC,GAAE,cAAiC,EAC7C,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,EAC5B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CAkG3B;AAkeD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;gBAqB7B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM;IAiBvB,IAAI,SAAS,uBAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;;IA0EnC;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS;IAcjE,IAAI;IA2CJ,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB;CAiDtE;AAunBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}

package/out/zero-cache/src/services/change-source/pg/change-source.js CHANGED Viewed

@@ -18,14 +18,15 @@ import { fromBigInt, toBigInt, toStateVersionString } from "./lsn.js";
 import { getPublicationInfo } from "./schema/published.js";
 import { replicationEventSchema } from "./schema/ddl.js";
 import { validate } from "./schema/validation.js";
-import { dropShard, getInternalShardConfig, getReplicaAtVersion, internalPublicationPrefix, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotPrefix } from "./schema/shard.js";
+import { dropShard, getActiveReplicas, getInternalShardConfig, getReplicaAtVersion, internalPublicationPrefix, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotPrefix } from "./schema/shard.js";
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
+import { restoreReplica } from "../common/replica-restore.js";
 import { initReplica } from "../common/replica-schema.js";
 import "../../../types/pg-versions.js";
 import { BackfillManager } from "../common/backfill-manager.js";
 import { ChangeStreamMultiplexer } from "../common/change-stream-multiplexer.js";
 import { dropOldReplicasAndSlots } from "./replication-slots.js";
-import { updateShardSchema } from "./schema/init.js";
+import { ensureShardSchema } from "./schema/init.js";
 import { initialSync } from "./initial-sync.js";
 import { streamBackfill } from "./backfill-stream.js";
 import { subscribe } from "./logical-replication/stream.js";
@@ -40,25 +41,57 @@ var REPLICA_SLOT_CLEANUP_INTERVAL_MS = 3e4;
 * replica, before streaming changes from the corresponding logical replication
 * stream.
 */
-async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbFile, syncOptions, context, lagReportIntervalMs = 0, streamInboundTimeoutMs) {
-	await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, (log, tx) => initialSync(log, shard, tx, upstreamURI, syncOptions, context));
-	const replica = new Database(lc, replicaDbFile);
-	const subscriptionState = getSubscriptionStateAndContext(new StatementRunner(replica));
-	replica.close();
+async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbFile, syncOptions, context, lagReportIntervalMs = 0, restoreOptions = {}, { backupV5 } = { backupV5: true }, purgeLock, streamInboundTimeoutMs) {
 	const db = await connectPgClient(lc, upstreamURI, "change-source-init");
 	try {
+		await ensureShardSchema(lc, db, shard);
+		const restoredReplica = await selectAndRestoreReplica(lc, db, shard, replicaDbFile, restoreOptions);
+		let initialSyncedReplica;
+		await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, async (log, tx) => {
+			purgeLock?.release();
+			initialSyncedReplica = await initialSync(log, shard, tx, upstreamURI, syncOptions, context, { backupV5 });
+		});
+		const replica = new Database(lc, replicaDbFile);
+		const subscriptionState = getSubscriptionStateAndContext(new StatementRunner(replica));
+		replica.close();
+		const upstreamReplica = await checkAndUpdateUpstream(lc, db, shard, subscriptionState, (initialSyncedReplica ?? restoredReplica)?.id);
+		const backupPath = initialSyncedReplica ? initialSyncedReplica.backupPath : backupV5 ? String(Date.now()) : restoredReplica?.backupPath ?? null;
 		return {
 			subscriptionState,
-			changeSource: new PostgresChangeSource(lc, upstreamURI, shard, await checkAndUpdateUpstream(lc, db, shard, subscriptionState), context, lagReportIntervalMs, syncOptions.textCopy, streamInboundTimeoutMs)
+			changeSource: new PostgresChangeSource(lc, upstreamURI, shard, upstreamReplica, {
+				backupPath,
+				backupV5
+			}, context, lagReportIntervalMs, syncOptions.textCopy, streamInboundTimeoutMs),
+			destinationBackupURL: backupPath && restoreOptions.litestream?.backupURL ? new URL(backupPath, restoreOptions.litestream.backupURL).toString() : restoreOptions.litestream?.backupURL,
+			replicaID: upstreamReplica.id,
+			waitForBackupBeforeServing: initialSyncedReplica !== void 0 || backupPath !== (restoredReplica?.backupPath ?? null)
 		};
 	} finally {
 		await db.end();
 	}
 }
-async function checkAndUpdateUpstream(lc, sql, shard, { replicaVersion, publications: subscribed, initialSyncContext }) {
-	await updateShardSchema(lc, sql, shard, replicaVersion);
-	const upstreamReplica = await getReplicaAtVersion(lc, sql, shard, replicaVersion, initialSyncContext);
-	if (!upstreamReplica) throw new AutoResetSignal(`No replication slot for replica at version ${replicaVersion}`);
+async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream, constraints }) {
+	const replicas = (await getActiveReplicas(lc, sql, shard)).filter(({ generation }) => generation === (constraints?.replicaVersion ?? generation));
+	if (replicas.length === 0) {
+		lc.info?.(`no suitable replicas to restore from`, { replicas });
+		return;
+	}
+	const [replica] = replicas;
+	if (litestream?.backupURL) {
+		const { backupURL: backupBaseURL } = litestream;
+		const { slot, backupPath, confirmedFlushLsn } = replica;
+		const backupURL = new URL(backupPath ?? "", backupBaseURL).toString();
+		lc.info?.(`restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`, { replicas });
+		await restoreReplica(lc, {
+			...litestream,
+			backupURL
+		}, replicaFile, constraints);
+	}
+	return replica;
+}
+async function checkAndUpdateUpstream(lc, sql, shard, { replicaVersion, publications: subscribed, initialSyncContext }, replicaID) {
+	const upstreamReplica = await getReplicaAtVersion(lc, sql, shard, replicaVersion, replicaID, initialSyncContext);
+	if (!upstreamReplica) throw new AutoResetSignal(`No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`);
 	const requested = shard.publications.toSorted();
 	const replicated = upstreamReplica.publications.filter((p) => !p.startsWith(internalPublicationPrefix(shard))).sort();
 	if (!deepEqual(requested, replicated)) {
@@ -91,12 +124,13 @@ var PostgresChangeSource = class {
 	#upstreamUri;
 	#shard;
 	#replica;
+	#backupOptions;
 	#context;
 	#lagReporter;
 	#textCopy;
 	#streamInboundTimeoutMs;
 	#stopped = false;
-	constructor(lc, upstreamUri, shard, replica, context, lagReportIntervalMs, textCopy, streamInboundTimeoutMs) {
+	constructor(lc, upstreamUri, shard, replica, backupOptions, context, lagReportIntervalMs, textCopy, streamInboundTimeoutMs) {
 		this.#lc = lc.withContext("component", "change-source");
 		this.#db = pgClient(lc, upstreamUri, "replication-monitor", {
 			max: 1,
@@ -105,6 +139,7 @@ var PostgresChangeSource = class {
 		this.#upstreamUri = upstreamUri;
 		this.#shard = shard;
 		this.#replica = replica;
+		this.#backupOptions = backupOptions;
 		this.#context = context;
 		this.#textCopy = textCopy ?? false;
 		this.#streamInboundTimeoutMs = streamInboundTimeoutMs;
@@ -129,7 +164,7 @@ var PostgresChangeSource = class {
 		return null;
 	}
 	async startStream(clientWatermark, backfillRequests = []) {
-		await this.#stopExistingReplicationSlotSubscriber();
+		await this.#takeoverReplicationSlot();
 		const config = await getInternalShardConfig(this.#db, this.#shard);
 		const { slot } = this.#replica;
 		this.#lc.info?.(`starting replication stream@${slot}`);
@@ -207,7 +242,7 @@ var PostgresChangeSource = class {
 				changes.fail(err);
 			}
 		})();
-		this.#lc.info?.(`started replication stream@${slot} from ${clientWatermark} (replicaVersion: ${this.#replica.version})`);
+		this.#lc.info?.(`started replication stream@${slot} from ${clientWatermark} (replicaVersion: ${this.#replica.generation})`);
 		return {
 			changes: changes.asSource(),
 			acks: { push: (status) => acker.ack(status[2].watermark) }
@@ -215,17 +250,29 @@ var PostgresChangeSource = class {
 	}
 	async #logCurrentReplicaInfo() {
 		try {
-			const replica = await getReplicaAtVersion(this.#lc, this.#db, this.#shard, this.#replica.version);
-			if (replica) this.#lc.info?.(`Shutdown signal from replica@${this.#replica.version}: ${stringify(replica.subscriberContext)}`);
+			const replica = await getReplicaAtVersion(this.#lc, this.#db, this.#shard, this.#replica.generation, this.#replica.id);
+			if (replica) this.#lc.info?.(`Shutdown signal from replica@${this.#replica.generation}: ${stringify(replica.subscriberContext)}`);
 		} catch (e) {
 			this.#lc.warn?.(`error logging replica info`, e);
 		}
 	}
 	/**
-	* Stops replication slots associated with this shard, and asynchronously
-	* runs a cleanup task that drops older replicas and slots.
+	* In RMv1, a single replication slot is taken over by the next
+	* replication-manager, signaling the old one to shut down.
+	*
+	* With litestream v5, the two replication-managers must replicate to unique
+	* backupPaths, as is required by the LTX backup schema.
+	*
+	* Thus, in addition to taking over the replication slot, the
+	* replication-manager also updates the `replicas` table with its
+	* `backupPath` so that future RM's restore from that path instead
+	* of the vestigial path of the previous replication-manager.
+	*
+	* In RMv2, each replication-manager will have its own slot and
+	* row in the `replicas` table, so this "takeover" will be unnecessary
+	* (but a harmless no-op).
 	*/
-	async #stopExistingReplicationSlotSubscriber() {
+	async #takeoverReplicationSlot() {
 		const sql = this.#db;
 		const { id: replicaID, slot } = this.#replica;
 		const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;
@@ -237,7 +284,7 @@ var PostgresChangeSource = class {
 		if (result.length === 0) {
 			const slotExpression = replicationSlotPrefix(this.#shard);
 			const replicas = await sql`
-        SELECT id, rank, slot, version, "initialSyncContext", "subscriberContext"
+        SELECT id, rank, slot, generation, "initialSyncContext", "subscriberContext"
           FROM ${sql(replicasTable)} ORDER BY rank DESC`;
 			const slots = await sql`
         SELECT slot_name as slot, active, active_pid as pid
@@ -253,7 +300,9 @@ var PostgresChangeSource = class {
 		this.#lc.info?.(`terminated replication slots: ${JSON.stringify(result)}`);
 		await sql`
       UPDATE ${sql(replicasTable)}
-        SET "subscriberContext" = ${this.#context}
+        SET "subscriberContext" = ${this.#context},
+            "backupPath" = ${this.#backupOptions.backupPath},
+            "backupV5" = ${this.#backupOptions.backupV5}
         WHERE id = ${replicaID}`;
 		this.#cleanUpOlderReplicasAndSlots();
 	}
@@ -393,7 +442,10 @@ var LagReporter = class LagReporter {
 			sendTimeMs: now,
 			commitTimeMs
 		});
-		return { nextSendTimeMs: now };
+		return {
+			firstCommitTimeMs: commitTimeMs,
+			nextSendTimeMs: now
+		};
 	}
 	/**
 	* In Postgres < 17, the pg_logical_emit_message lacks an immediate "flush"