@rocicorp/zero 1.11.0-head-eb784601-20261007 → 1.11.0-head-e95cccc9-20261007
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +2 -1
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts +5 -0
- package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-restore.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts +26 -1
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js +37 -5
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts +7 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +31 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +5 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +1 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +7 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +23 -11
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/package.json +1 -1
package/out/zero/package.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
var package_default = {
|
|
2
2
|
name: "@rocicorp/zero",
|
|
3
|
-
version: "1.11.0-head-
|
|
3
|
+
version: "1.11.0-head-e95cccc9-20261007",
|
|
4
4
|
description: "Zero is a web framework for serverless web development.",
|
|
5
5
|
homepage: "https://zero.rocicorp.dev",
|
|
6
6
|
bugs: { "url": "https://bugs.rocicorp.dev" },
|
package/out/zero/package.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.11.0-head-
|
|
1
|
+
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.11.0-head-e95cccc9-20261007\",\n \"description\": \"Zero is a web framework for serverless web development.\",\n \"homepage\": \"https://zero.rocicorp.dev\",\n \"bugs\": {\n \"url\": \"https://bugs.rocicorp.dev\"\n },\n \"license\": \"Apache-2.0\",\n \"author\": \"Rocicorp, Inc.\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/rocicorp/mono.git\",\n \"directory\": \"packages/zero\"\n },\n \"bin\": {\n \"analyze-query\": \"./out/zero/src/analyze-query.js\",\n \"ast-to-zql\": \"./out/zero/src/ast-to-zql.js\",\n \"transform-query\": \"./out/zero/src/transform-query.js\",\n \"zero-build-schema\": \"./out/zero/src/build-schema.js\",\n \"zero-cache\": \"./out/zero/src/cli.js\",\n \"zero-cache-dev\": \"./out/zero/src/zero-cache-dev.js\",\n \"zero-deploy-permissions\": \"./out/zero/src/deploy-permissions.js\",\n \"zero-out\": \"./out/zero/src/zero-out.js\"\n },\n \"files\": [\n \"out\",\n \"!*.tsbuildinfo\"\n ],\n \"type\": \"module\",\n \"main\": \"out/zero/src/zero.js\",\n \"module\": \"out/zero/src/zero.js\",\n \"types\": \"out/zero/src/zero.d.ts\",\n \"exports\": {\n \".\": {\n \"types\": \"./out/zero/src/zero.d.ts\",\n \"default\": \"./out/zero/src/zero.js\"\n },\n \"./analyze\": {\n \"types\": \"./out/zero/src/analyze.d.ts\",\n \"default\": \"./out/zero/src/analyze.js\"\n },\n \"./bindings\": {\n \"types\": \"./out/zero/src/bindings.d.ts\",\n \"default\": \"./out/zero/src/bindings.js\"\n },\n \"./change-protocol/v0\": {\n \"types\": \"./out/zero/src/change-protocol/v0.d.ts\",\n \"default\": \"./out/zero/src/change-protocol/v0.js\"\n },\n \"./expo-sqlite\": {\n \"types\": \"./out/zero/src/expo-sqlite.d.ts\",\n \"default\": \"./out/zero/src/expo-sqlite.js\"\n },\n \"./op-sqlite\": {\n \"types\": \"./out/zero/src/op-sqlite.d.ts\",\n \"default\": \"./out/zero/src/op-sqlite.js\"\n },\n \"./pg\": {\n \"types\": \"./out/zero/src/pg.d.ts\",\n \"default\": \"./out/zero/src/pg.js\"\n },\n \"./react\": {\n \"types\": \"./out/zero/src/react.d.ts\",\n \"default\": \"./out/zero/src/react.js\"\n },\n \"./react-native\": {\n \"types\": \"./out/zero/src/react-native.d.ts\",\n \"default\": \"./out/zero/src/react-native.js\"\n },\n \"./server\": {\n \"types\": \"./out/zero/src/server.d.ts\",\n \"default\": \"./out/zero/src/server.js\"\n },\n \"./server/adapters/drizzle\": {\n \"types\": \"./out/zero/src/adapters/drizzle.d.ts\",\n \"default\": \"./out/zero/src/adapters/drizzle.js\"\n },\n \"./server/adapters/kysely\": {\n \"types\": \"./out/zero/src/adapters/kysely.d.ts\",\n \"default\": \"./out/zero/src/adapters/kysely.js\"\n },\n \"./server/adapters/prisma\": {\n \"types\": \"./out/zero/src/adapters/prisma.d.ts\",\n \"default\": \"./out/zero/src/adapters/prisma.js\"\n },\n \"./server/adapters/pg\": {\n \"types\": \"./out/zero/src/adapters/pg.d.ts\",\n \"default\": \"./out/zero/src/adapters/pg.js\"\n },\n \"./server/adapters/postgresjs\": {\n \"types\": \"./out/zero/src/adapters/postgresjs.d.ts\",\n \"default\": \"./out/zero/src/adapters/postgresjs.js\"\n },\n \"./solid\": {\n \"types\": \"./out/zero/src/solid.d.ts\",\n \"default\": \"./out/zero/src/solid.js\"\n },\n \"./sqlite\": {\n \"types\": \"./out/zero/src/sqlite.d.ts\",\n \"default\": \"./out/zero/src/sqlite.js\"\n },\n \"./zqlite\": {\n \"types\": \"./out/zero/src/zqlite.d.ts\",\n \"default\": \"./out/zero/src/zqlite.js\"\n }\n },\n \"scripts\": {\n \"build\": \"node --experimental-strip-types --no-warnings tool/build.ts\",\n \"build:watch\": \"node --experimental-strip-types --no-warnings tool/build.ts --watch\",\n \"check-types\": \"tsc -p tsconfig.client.json && tsc -p tsconfig.server.json\",\n \"check-types:client:watch\": \"tsc -p tsconfig.client.json --watch\",\n \"check-types:server:watch\": \"tsc -p tsconfig.server.json --watch\",\n \"format\": \"oxfmt .\",\n \"check-format\": \"oxfmt --check .\",\n \"lint\": \"oxlint --quiet --config ../../oxlint.config.ts src/\",\n \"docs\": \"node --experimental-strip-types --no-warnings tool/generate-docs.ts\",\n \"docs:server\": \"node --watch --experimental-strip-types --no-warnings tool/generate-docs.ts --server\",\n \"fmt\": \"oxfmt .\",\n \"check-fmt\": \"oxfmt --check .\"\n },\n \"dependencies\": {\n \"@badrap/valita\": \"0.5.5\",\n \"@databases/escape-identifier\": \"^1.0.3\",\n \"@databases/sql\": \"^3.3.0\",\n \"@dotenvx/dotenvx\": \"^1.39.0\",\n \"@drdgvhbh/postgres-error-codes\": \"^0.0.6\",\n \"@fastify/cors\": \"^10.0.0\",\n \"@fastify/websocket\": \"^11.0.0\",\n \"@google-cloud/precise-date\": \"^4.0.0\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/api-logs\": \"^0.218.0\",\n \"@opentelemetry/auto-instrumentations-node\": \"^0.76.0\",\n \"@opentelemetry/exporter-metrics-otlp-http\": \"^0.218.0\",\n \"@opentelemetry/resources\": \"^2.7.1\",\n \"@opentelemetry/sdk-metrics\": \"^2.7.1\",\n \"@opentelemetry/sdk-node\": \"^0.218.0\",\n \"@opentelemetry/sdk-trace-node\": \"^2.7.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.41.1\",\n \"@postgresql-typed/oids\": \"^0.2.0\",\n \"@rocicorp/lock\": \"^1.0.4\",\n \"@rocicorp/logger\": \"^6.1.0\",\n \"@rocicorp/resolver\": \"^1.0.2\",\n \"@rocicorp/zero-sqlite3\": \"^1.1.4\",\n \"@standard-schema/spec\": \"^1.0.0\",\n \"@types/basic-auth\": \"^1.1.8\",\n \"@types/ws\": \"^8.5.12\",\n \"basic-auth\": \"^2.0.1\",\n \"chalk-template\": \"^1.1.0\",\n \"chokidar\": \"^4.0.1\",\n \"cloudevents\": \"^10.0.0\",\n \"command-line-args\": \"^6.0.1\",\n \"command-line-usage\": \"^7.0.3\",\n \"compare-utf8\": \"^0.2.3\",\n \"defu\": \"^6.1.4\",\n \"eventemitter3\": \"^5.0.1\",\n \"fastify\": \"^5.0.0\",\n \"is-in-subnet\": \"^4.0.1\",\n \"jose\": \"^5.9.3\",\n \"json-custom-numbers\": \"^3.1.1\",\n \"kasi\": \"^1.1.0\",\n \"nanoid\": \"^5.1.2\",\n \"oxfmt\": \"^0.65.0\",\n \"parse-prometheus-text-format\": \"^1.1.1\",\n \"pg-format\": \"npm:pg-format-fix@^1.0.5\",\n \"postgres\": \"3.4.7\",\n \"semver\": \"^7.5.4\",\n \"url-pattern\": \"^1.0.3\",\n \"urlpattern-polyfill\": \"^10.1.0\",\n \"ws\": \"^8.18.1\"\n },\n \"devDependencies\": {\n \"@op-engineering/op-sqlite\": \">=15\",\n \"analyze-query\": \"workspace:*\",\n \"ast-to-zql\": \"workspace:*\",\n \"expo-sqlite\": \">=15\",\n \"oxc-parser\": \"^0.129.0\",\n \"replicache\": \"workspace:*\",\n \"shared\": \"workspace:*\",\n \"syncpack\": \"^14.3.0\",\n \"typedoc\": \"^0.28.17\",\n \"typescript\": \"~7.0.2\",\n \"vite\": \"^8.0.13\",\n \"vitest\": \"^5.0.0\",\n \"zero-cache\": \"workspace:*\",\n \"zero-client\": \"workspace:*\",\n \"zero-pg\": \"workspace:*\",\n \"zero-react\": \"workspace:*\",\n \"zero-server\": \"workspace:*\",\n \"zero-solid\": \"workspace:*\",\n \"zqlite\": \"workspace:*\"\n },\n \"engines\": {\n \"node\": \">=22\"\n },\n \"packageManager\": \"pnpm@11.11.0\"\n}\n"],"mappings":""}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AA4CA,OAAO,EAML,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAW/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AA4CA,OAAO,EAML,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAW/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAmaf"}
|
|
@@ -92,7 +92,7 @@ async function runWorker(parent, env, ...argv) {
|
|
|
92
92
|
};
|
|
93
93
|
try {
|
|
94
94
|
const { pgReplicationEpoch: epoch, pgReplicationSlotPerReplica: slotPerReplica, pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs } = upstream;
|
|
95
|
-
const { changeSource, subscriptionState, destinationBackupURL, replicaID, waitForBackupBeforeServing, newBackupLineage: initNewBackupLineage, walKeeper: initWalKeeper } = upstream.type === "pg" ? await initializePostgresChangeSource(lc, upstream.db, shard, replica.file, {
|
|
95
|
+
const { changeSource, subscriptionState, destinationBackupURL, replicaID, waitForBackupBeforeServing, newBackupLineage: initNewBackupLineage, walKeeper: initWalKeeper, pgChangeLogBehindSlot } = upstream.type === "pg" ? await initializePostgresChangeSource(lc, upstream.db, shard, replica.file, {
|
|
96
96
|
...initialSync,
|
|
97
97
|
replicationSlotFailover: upstream.pgReplicationSlotFailover,
|
|
98
98
|
installPartialIndexTriggers: upstream.pgPartialIndexTriggers
|
|
@@ -110,6 +110,7 @@ async function runWorker(parent, env, ...argv) {
|
|
|
110
110
|
replicaFile: replica.file
|
|
111
111
|
} : null, purgeLock, autoReset ?? false, {
|
|
112
112
|
pgChangeLogEnabled,
|
|
113
|
+
pgChangeLogBehindSlot: pgChangeLogBehindSlot ?? false,
|
|
113
114
|
backPressureLimitHeapProportion,
|
|
114
115
|
flowControlConsensusTimeoutProportion,
|
|
115
116
|
flowControlSlowSubscriberGracePeriodMs: flowControlSlowSubscriberGracePeriodSeconds > 0 ? flowControlSlowSubscriberGracePeriodSeconds * 1e3 : void 0,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {\n PurgeLocker,\n type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {deleteLitestreamMetaDir} from '../services/litestream/commands.ts';\nimport type {WalKeeper} from '../services/litestream/wal-keeper.ts';\nimport {\n changeLogFileName,\n deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n broadcastWorker,\n childWorker,\n parentWorker,\n singleProcessMode,\n type ProfileResponseMessage,\n type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n installProfileHandler(parent, 'change-streamer');\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodSeconds,\n pgChangeLogEnabled,\n sqliteChangeLogMode,\n sqliteChangeLogReadPercent,\n sqliteChangeLogColdReadPercent,\n sqliteChangeLogComparePercent,\n sqliteChangeLogRetentionMs,\n sqliteChangeLogReadBatchRows,\n sqliteChangeLogPurgeBatchRows,\n sqliteChangeLogBarrierTimeoutMs,\n },\n autoReset,\n replicationLag,\n litestream,\n upstream,\n change,\n replica,\n initialSync,\n keepaliveTimeoutMs,\n sqliteCorruptionChecks,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n lc = createLogContext(config, 'change-streamer');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer replica',\n dbPath: replica.file,\n },\n sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Startup-time client used for for change-streamer initialization\n // and handoff / takeover. Steady-state clients are managed by the\n // change-streamer (Storer) implementation.\n const changeDB = await connectPgClient(\n lc,\n change.db,\n 'change-streamer-init',\n {max: 5},\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date.\n await initChangeStreamerSchema(lc, changeDB, shard);\n\n let purgeLock = null as PurgeLock | null;\n let changeStreamer: ChangeStreamerService | undefined;\n let backupURL: string | undefined;\n\n const context = getServerContext(config);\n const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n // The modes are cumulative, so `serve` implies `compare`.\n const sqliteChangeLogComparing =\n sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n if (sqliteChangeLogEnabled) {\n const changeLogFile = changeLogFileName(replica.file);\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer change-log',\n dbPath: changeLogFile,\n },\n sqliteCorruptionChecks,\n );\n // The log's bytes are accounted for in the process that writes them. This\n // is local disk only — the log is excluded from the litestream backup — and\n // it is a whole-database total, because a table that left the replica left\n // through its wal: the replica's footprint is what shrank and this is where\n // it went.\n getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n description:\n `The SQLite change log's total on-disk footprint: its main db file ` +\n `plus its wal sidecars.`,\n unit: 'By',\n }).addCallback(async o =>\n o.observe(await sqliteFileBytes(lc, changeLogFile)),\n );\n }\n\n let waitForFirstBackupBeforeServing = false;\n let newBackupLineage = false;\n let walKeeper: WalKeeper | undefined;\n for (const first of [true, false]) {\n // Resources acquired by an initialization attempt (e.g. a claimed\n // replication slot, the purge lock) to be released if the attempt fails.\n const cleanup = new InitCleanup(lc);\n // When restoring from litestream, a lock is acquired to prevent change-log\n // purges. This ensures that (this) change-streamer will be able to resume\n // from the backup. The lock is acquired by the change source initialization\n // just before restoring (i.e. after a replication slot is created, if\n // applicable), and released once this change-streamer takes over the\n // change-log.\n const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n pgChangeLogEnabled && litestream.backupURL && litestream.executable\n ? async (slotWatermark?: string) => {\n const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n slotWatermark,\n );\n purgeLock = lock === 'behind-slot' ? null : lock;\n if (purgeLock) {\n const acquired = purgeLock;\n cleanup.onFailure('purge lock', () => {\n purgeLock = null;\n return acquired.release();\n });\n }\n return lock;\n }\n : undefined;\n\n const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {\n pgReplicationEpoch: epoch,\n pgReplicationSlotPerReplica: slotPerReplica,\n pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n } = upstream;\n const {\n changeSource,\n subscriptionState,\n destinationBackupURL,\n replicaID,\n waitForBackupBeforeServing,\n newBackupLineage: initNewBackupLineage,\n walKeeper: initWalKeeper,\n } = upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n },\n context,\n replicationLag.reportIntervalMs,\n restoreOptions,\n {\n epoch,\n slotPerReplica,\n inactiveReplicaGracePeriodMs,\n backupV5: litestream.backupUsingV5,\n },\n upstream.pgStreamInboundTimeoutMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n restoreOptions,\n );\n walKeeper = initWalKeeper;\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n destinationBackupURL\n ? {\n backupURL: destinationBackupURL,\n litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n replicaFile: replica.file,\n }\n : null,\n purgeLock,\n autoReset ?? false,\n {\n pgChangeLogEnabled,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n flowControlSlowSubscriberGracePeriodSeconds > 0\n ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n : undefined,\n statementTimeoutMs: change.statementTimeoutMs,\n changeLogBatchSize: change.logBatchSize,\n sqliteCatchup: {\n changeLogFile: changeLogFileName(replica.file),\n readBatchRows: sqliteChangeLogReadBatchRows,\n barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n },\n // The presence of these options is the writer's gate. This process\n // performs the restore and the initial sync, so the replica's\n // identity is in hand at the moment the log is opened.\n sqliteChangeLogWriter: sqliteChangeLogEnabled\n ? {\n replicaFile: replica.file,\n identity: {\n // RMv2 supplies the epoch; until then the generation and the\n // replica ID are the whole of the identity.\n epoch: null,\n generation: subscriptionState.replicaVersion,\n replicaID,\n },\n }\n : undefined,\n // The purge scheduler shares the writer's gate -- mode, not the\n // read path -- because `write` mode, with no read selector at all,\n // is the configuration it ships in.\n sqliteChangeLogPurge: sqliteChangeLogEnabled\n ? {\n retentionMs: sqliteChangeLogRetentionMs,\n batchRows: sqliteChangeLogPurgeBatchRows,\n }\n : undefined,\n // Compare mode runs both advisory checks. Postgres remains authoritative.\n sqliteChangeLogCompare:\n pgChangeLogEnabled && sqliteChangeLogComparing\n ? {\n replicaFile: replica.file,\n comparePercent: sqliteChangeLogComparePercent,\n retentionMs: sqliteChangeLogRetentionMs,\n readBatchRows: sqliteChangeLogReadBatchRows,\n }\n : undefined,\n // Slice 11 lands dark by default: serve mode constructs the stable\n // router, while readPercent=0 keeps every catchup on PG and emits\n // eligibility metrics before any canary traffic is enabled.\n sqliteChangeLogServe:\n sqliteChangeLogMode === 'serve'\n ? {\n readPercent: sqliteChangeLogReadPercent,\n coldReadPercent: sqliteChangeLogColdReadPercent,\n retentionMs: sqliteChangeLogRetentionMs,\n }\n : undefined,\n },\n setTimeout,\n );\n backupURL = destinationBackupURL;\n waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n newBackupLineage = initNewBackupLineage;\n break;\n } catch (e) {\n // Release the resources acquired by the failed attempt, e.g.:\n // * The claimed replication slot, which would otherwise remain active\n // indefinitely, preventing a retry from resuming (or forking) it.\n // * The purge lock. This is safe because the purge lock exists to\n // preserve change-log entries so the new change-streamer can resume\n // from the backup replica's watermark. An AutoResetSignal means we\n // can't resume from the backup replica (e.g. its replication slot is\n // gone), so the change-log entries the lock was protecting are no\n // longer needed. The retry performs a fresh initial sync with a new\n // replication slot, independent of the old change-log. Releasing is\n // also necessary to avoid a self-deadlock when CHANGE_DB ==\n // UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n // transactions to finish, including this lock's open transaction.\n // * The WAL keeper of a restore that prepared a backup lineage to\n // continue, which would otherwise hold the (deleted) replica open.\n await cleanup.release();\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // The change log carries the identity of the replica it was written\n // beside, and the retry performs a fresh initial sync with a new\n // replicaVersion. Reconciliation would catch that on its own (as\n // 'identity-mismatch') but only after the writer opened a file that is\n // known here to be garbage.\n deleteChangeLogDB(replica.file);\n continue; // execute again with a fresh initial-sync\n }\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n const processes = new ProcessManager(lc, parent);\n const profileSubWorkers: Worker[] = [];\n const {promise: replicatorReady, resolve} = resolver();\n if (!backupURL) {\n resolve(); // No backup replicator to wait for, so resolve immediately.\n } else {\n lc.info?.('setting up backup to', backupURL);\n config.litestream.backupURLOverride = backupURL;\n if (newBackupLineage) {\n deleteLitestreamMetaDir(replica.file);\n }\n // Start a backup replicator, which starts up the corresponding\n // litestream backup process.\n const backupReplicator = processes\n .addWorker(\n childWorker(\n REPLICATOR_URL,\n {...env, ZERO_LITESTREAM_BACKUP_URL_OVERRIDE: backupURL},\n 'backup' satisfies ReplicaFileMode,\n ),\n 'supporting',\n 'backup-replicator',\n )\n .onceMessageType('ready', () => resolve());\n profileSubWorkers.push(backupReplicator);\n // Relay profileResponse messages from backup-replicator up to parent\n backupReplicator.onMessageType<ProfileResponseMessage>(\n 'profileResponse',\n res => parent.send(['profileResponse', res]),\n );\n }\n\n const backupMonitor = createBackupCleanupMonitor({\n lc,\n config,\n replicaFile: replica.file,\n changeStreamer,\n });\n\n let backupReady = promiseVoid;\n if (waitForFirstBackupBeforeServing) {\n const start = performance.now();\n lc.info?.(`awaiting initial backup ...`);\n\n backupReady = backupMonitor.firstBackupReceived().then(() => {\n const elapsed = performance.now() - start;\n lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n });\n }\n // By the initial backup, litestream holds the restored WAL itself.\n void backupReady.then(\n () => walKeeper?.release('initial backup confirmed'),\n () => walKeeper?.release('initial backup failed'),\n );\n\n // In RMv2, readiness additionally waits for the backup-replicator to catch\n // up to the replication stream. This is not done in RMv1 (i.e. when the\n // PG change-log is enabled), in which the change-streamer itself is only\n // started after the readinessGate (plus a startup delay), and thus the\n // backup-replicator cannot catch up before then.\n const readinessGate = pgChangeLogEnabled\n ? backupReady\n : Promise.all([backupReady, replicatorReady]);\n\n // Create the broadcast facade once: each broadcastWorker() adds permanent\n // 'message' forwarders to every sub worker, so creating one per /profz\n // request would leak a forwarder per request.\n const profileWorker =\n profileSubWorkers.length > 0\n ? broadcastWorker(profileSubWorkers)\n : undefined;\n const getProfileWorker = profileWorker\n ? () => Promise.resolve(profileWorker)\n : undefined;\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n {\n port,\n keepaliveTimeoutMs,\n // The startup delay is only relevant when taking over the PG change-log\n // (RMv1 and RMv1.5), and is disabled for RMv2.\n startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n readinessGate,\n config,\n getProfileWorker,\n },\n parent,\n changeStreamer,\n );\n\n void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer after a delay to ensure that routing\n // routing elements have registered the server before the\n // change-streamer takes over the replication slot.\n // TODO: Remove this delay and start the changeStreamer normally here once\n // transitioned to RMv2, since changeStreamer startup will no longer\n // disrupt an existing task.\n try {\n await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n } catch (err) {\n processes.logErrorAndExit(err, 'change-streamer');\n } finally {\n walKeeper?.release('shutting down');\n await processes.shutdown();\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () =>\n runWorker(\n must(parentWorker),\n process.env,\n ...process.argv.slice(2),\n ).catch(async e => {\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n throw e;\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAElD,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,IAAI;CACJ,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,sBAAsB,WAAW,aAAa,WAAW,aACrD,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,WAAW,kBACT,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;GACvB,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GACJ,YAAY;GAEZ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE;IACA;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;GACR,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,mBAAmB;GACnB;EACF,SAAS,GAAG;GAgBV,MAAM,QAAQ,QAAQ;GACtB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,MAAM,EAAC,SAAS,iBAAiB,YAAW,SAAS;CACrD,IAAI,CAAC,WACH,QAAQ;MACH;EACL,GAAG,OAAO,wBAAwB,SAAS;EAC3C,OAAO,WAAW,oBAAoB;EACtC,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAItC,MAAM,mBAAmB,UACtB,UACC,YACE,gBACA;GAAC,GAAG;GAAK,qCAAqC;EAAS,GACvD,QACF,GACA,cACA,mBACF,EACC,gBAAgB,eAAe,QAAQ,CAAC;EAC3C,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;CACF;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,cAAc;CAClB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAc,cAAc,oBAAoB,EAAE,WAAW;GAC3D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,YAAiB,WACT,WAAW,QAAQ,0BAA0B,SAC7C,WAAW,QAAQ,uBAAuB,CAClD;CAOA,MAAM,gBAAgB,qBAClB,cACA,QAAQ,IAAI,CAAC,aAAa,eAAe,CAAC;CAK9C,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,WAAW,QAAQ,eAAe;EAClC,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
|
|
1
|
+
{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {\n PurgeLocker,\n type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {deleteLitestreamMetaDir} from '../services/litestream/commands.ts';\nimport type {WalKeeper} from '../services/litestream/wal-keeper.ts';\nimport {\n changeLogFileName,\n deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n broadcastWorker,\n childWorker,\n parentWorker,\n singleProcessMode,\n type ProfileResponseMessage,\n type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n installProfileHandler(parent, 'change-streamer');\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodSeconds,\n pgChangeLogEnabled,\n sqliteChangeLogMode,\n sqliteChangeLogReadPercent,\n sqliteChangeLogColdReadPercent,\n sqliteChangeLogComparePercent,\n sqliteChangeLogRetentionMs,\n sqliteChangeLogReadBatchRows,\n sqliteChangeLogPurgeBatchRows,\n sqliteChangeLogBarrierTimeoutMs,\n },\n autoReset,\n replicationLag,\n litestream,\n upstream,\n change,\n replica,\n initialSync,\n keepaliveTimeoutMs,\n sqliteCorruptionChecks,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n lc = createLogContext(config, 'change-streamer');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer replica',\n dbPath: replica.file,\n },\n sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Startup-time client used for for change-streamer initialization\n // and handoff / takeover. Steady-state clients are managed by the\n // change-streamer (Storer) implementation.\n const changeDB = await connectPgClient(\n lc,\n change.db,\n 'change-streamer-init',\n {max: 5},\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date.\n await initChangeStreamerSchema(lc, changeDB, shard);\n\n let purgeLock = null as PurgeLock | null;\n let changeStreamer: ChangeStreamerService | undefined;\n let backupURL: string | undefined;\n\n const context = getServerContext(config);\n const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n // The modes are cumulative, so `serve` implies `compare`.\n const sqliteChangeLogComparing =\n sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n if (sqliteChangeLogEnabled) {\n const changeLogFile = changeLogFileName(replica.file);\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer change-log',\n dbPath: changeLogFile,\n },\n sqliteCorruptionChecks,\n );\n // The log's bytes are accounted for in the process that writes them. This\n // is local disk only — the log is excluded from the litestream backup — and\n // it is a whole-database total, because a table that left the replica left\n // through its wal: the replica's footprint is what shrank and this is where\n // it went.\n getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n description:\n `The SQLite change log's total on-disk footprint: its main db file ` +\n `plus its wal sidecars.`,\n unit: 'By',\n }).addCallback(async o =>\n o.observe(await sqliteFileBytes(lc, changeLogFile)),\n );\n }\n\n let waitForFirstBackupBeforeServing = false;\n let newBackupLineage = false;\n let walKeeper: WalKeeper | undefined;\n for (const first of [true, false]) {\n // Resources acquired by an initialization attempt (e.g. a claimed\n // replication slot, the purge lock) to be released if the attempt fails.\n const cleanup = new InitCleanup(lc);\n // When restoring from litestream, a lock is acquired to prevent change-log\n // purges. This ensures that (this) change-streamer will be able to resume\n // from the backup. The lock is acquired by the change source initialization\n // just before restoring (i.e. after a replication slot is created, if\n // applicable), and released once this change-streamer takes over the\n // change-log.\n const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n pgChangeLogEnabled && litestream.backupURL && litestream.executable\n ? async (slotWatermark?: string) => {\n const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n slotWatermark,\n );\n purgeLock = lock === 'behind-slot' ? null : lock;\n if (purgeLock) {\n const acquired = purgeLock;\n cleanup.onFailure('purge lock', () => {\n purgeLock = null;\n return acquired.release();\n });\n }\n return lock;\n }\n : undefined;\n\n const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {\n pgReplicationEpoch: epoch,\n pgReplicationSlotPerReplica: slotPerReplica,\n pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n } = upstream;\n const {\n changeSource,\n subscriptionState,\n destinationBackupURL,\n replicaID,\n waitForBackupBeforeServing,\n newBackupLineage: initNewBackupLineage,\n walKeeper: initWalKeeper,\n pgChangeLogBehindSlot,\n } = upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n },\n context,\n replicationLag.reportIntervalMs,\n restoreOptions,\n {\n epoch,\n slotPerReplica,\n inactiveReplicaGracePeriodMs,\n backupV5: litestream.backupUsingV5,\n },\n upstream.pgStreamInboundTimeoutMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n restoreOptions,\n );\n walKeeper = initWalKeeper;\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n destinationBackupURL\n ? {\n backupURL: destinationBackupURL,\n litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n replicaFile: replica.file,\n }\n : null,\n purgeLock,\n autoReset ?? false,\n {\n pgChangeLogEnabled,\n pgChangeLogBehindSlot: pgChangeLogBehindSlot ?? false,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n flowControlSlowSubscriberGracePeriodSeconds > 0\n ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n : undefined,\n statementTimeoutMs: change.statementTimeoutMs,\n changeLogBatchSize: change.logBatchSize,\n sqliteCatchup: {\n changeLogFile: changeLogFileName(replica.file),\n readBatchRows: sqliteChangeLogReadBatchRows,\n barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n },\n // The presence of these options is the writer's gate. This process\n // performs the restore and the initial sync, so the replica's\n // identity is in hand at the moment the log is opened.\n sqliteChangeLogWriter: sqliteChangeLogEnabled\n ? {\n replicaFile: replica.file,\n identity: {\n // RMv2 supplies the epoch; until then the generation and the\n // replica ID are the whole of the identity.\n epoch: null,\n generation: subscriptionState.replicaVersion,\n replicaID,\n },\n }\n : undefined,\n // The purge scheduler shares the writer's gate -- mode, not the\n // read path -- because `write` mode, with no read selector at all,\n // is the configuration it ships in.\n sqliteChangeLogPurge: sqliteChangeLogEnabled\n ? {\n retentionMs: sqliteChangeLogRetentionMs,\n batchRows: sqliteChangeLogPurgeBatchRows,\n }\n : undefined,\n // Compare mode runs both advisory checks. Postgres remains authoritative.\n sqliteChangeLogCompare:\n pgChangeLogEnabled && sqliteChangeLogComparing\n ? {\n replicaFile: replica.file,\n comparePercent: sqliteChangeLogComparePercent,\n retentionMs: sqliteChangeLogRetentionMs,\n readBatchRows: sqliteChangeLogReadBatchRows,\n }\n : undefined,\n // Slice 11 lands dark by default: serve mode constructs the stable\n // router, while readPercent=0 keeps every catchup on PG and emits\n // eligibility metrics before any canary traffic is enabled.\n sqliteChangeLogServe:\n sqliteChangeLogMode === 'serve'\n ? {\n readPercent: sqliteChangeLogReadPercent,\n coldReadPercent: sqliteChangeLogColdReadPercent,\n retentionMs: sqliteChangeLogRetentionMs,\n }\n : undefined,\n },\n setTimeout,\n );\n backupURL = destinationBackupURL;\n waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n newBackupLineage = initNewBackupLineage;\n break;\n } catch (e) {\n // Release the resources acquired by the failed attempt, e.g.:\n // * The claimed replication slot, which would otherwise remain active\n // indefinitely, preventing a retry from resuming (or forking) it.\n // * The purge lock. This is safe because the purge lock exists to\n // preserve change-log entries so the new change-streamer can resume\n // from the backup replica's watermark. An AutoResetSignal means we\n // can't resume from the backup replica (e.g. its replication slot is\n // gone), so the change-log entries the lock was protecting are no\n // longer needed. The retry performs a fresh initial sync with a new\n // replication slot, independent of the old change-log. Releasing is\n // also necessary to avoid a self-deadlock when CHANGE_DB ==\n // UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n // transactions to finish, including this lock's open transaction.\n // * The WAL keeper of a restore that prepared a backup lineage to\n // continue, which would otherwise hold the (deleted) replica open.\n await cleanup.release();\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // The change log carries the identity of the replica it was written\n // beside, and the retry performs a fresh initial sync with a new\n // replicaVersion. Reconciliation would catch that on its own (as\n // 'identity-mismatch') but only after the writer opened a file that is\n // known here to be garbage.\n deleteChangeLogDB(replica.file);\n continue; // execute again with a fresh initial-sync\n }\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n const processes = new ProcessManager(lc, parent);\n const profileSubWorkers: Worker[] = [];\n const {promise: replicatorReady, resolve} = resolver();\n if (!backupURL) {\n resolve(); // No backup replicator to wait for, so resolve immediately.\n } else {\n lc.info?.('setting up backup to', backupURL);\n config.litestream.backupURLOverride = backupURL;\n if (newBackupLineage) {\n deleteLitestreamMetaDir(replica.file);\n }\n // Start a backup replicator, which starts up the corresponding\n // litestream backup process.\n const backupReplicator = processes\n .addWorker(\n childWorker(\n REPLICATOR_URL,\n {...env, ZERO_LITESTREAM_BACKUP_URL_OVERRIDE: backupURL},\n 'backup' satisfies ReplicaFileMode,\n ),\n 'supporting',\n 'backup-replicator',\n )\n .onceMessageType('ready', () => resolve());\n profileSubWorkers.push(backupReplicator);\n // Relay profileResponse messages from backup-replicator up to parent\n backupReplicator.onMessageType<ProfileResponseMessage>(\n 'profileResponse',\n res => parent.send(['profileResponse', res]),\n );\n }\n\n const backupMonitor = createBackupCleanupMonitor({\n lc,\n config,\n replicaFile: replica.file,\n changeStreamer,\n });\n\n let backupReady = promiseVoid;\n if (waitForFirstBackupBeforeServing) {\n const start = performance.now();\n lc.info?.(`awaiting initial backup ...`);\n\n backupReady = backupMonitor.firstBackupReceived().then(() => {\n const elapsed = performance.now() - start;\n lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n });\n }\n // By the initial backup, litestream holds the restored WAL itself.\n void backupReady.then(\n () => walKeeper?.release('initial backup confirmed'),\n () => walKeeper?.release('initial backup failed'),\n );\n\n // In RMv2, readiness additionally waits for the backup-replicator to catch\n // up to the replication stream. This is not done in RMv1 (i.e. when the\n // PG change-log is enabled), in which the change-streamer itself is only\n // started after the readinessGate (plus a startup delay), and thus the\n // backup-replicator cannot catch up before then.\n const readinessGate = pgChangeLogEnabled\n ? backupReady\n : Promise.all([backupReady, replicatorReady]);\n\n // Create the broadcast facade once: each broadcastWorker() adds permanent\n // 'message' forwarders to every sub worker, so creating one per /profz\n // request would leak a forwarder per request.\n const profileWorker =\n profileSubWorkers.length > 0\n ? broadcastWorker(profileSubWorkers)\n : undefined;\n const getProfileWorker = profileWorker\n ? () => Promise.resolve(profileWorker)\n : undefined;\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n {\n port,\n keepaliveTimeoutMs,\n // The startup delay is only relevant when taking over the PG change-log\n // (RMv1 and RMv1.5), and is disabled for RMv2.\n startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n readinessGate,\n config,\n getProfileWorker,\n },\n parent,\n changeStreamer,\n );\n\n void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer after a delay to ensure that routing\n // routing elements have registered the server before the\n // change-streamer takes over the replication slot.\n // TODO: Remove this delay and start the changeStreamer normally here once\n // transitioned to RMv2, since changeStreamer startup will no longer\n // disrupt an existing task.\n try {\n await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n } catch (err) {\n processes.logErrorAndExit(err, 'change-streamer');\n } finally {\n walKeeper?.release('shutting down');\n await processes.shutdown();\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () =>\n runWorker(\n must(parentWorker),\n process.env,\n ...process.argv.slice(2),\n ).catch(async e => {\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n throw e;\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAElD,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,IAAI;CACJ,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,sBAAsB,WAAW,aAAa,WAAW,aACrD,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,WAAW,eACX,0BACE,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;GACvB,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GACJ,YAAY;GAEZ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE;IACA,uBAAuB,yBAAyB;IAChD;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;GACR,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,mBAAmB;GACnB;EACF,SAAS,GAAG;GAgBV,MAAM,QAAQ,QAAQ;GACtB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,MAAM,EAAC,SAAS,iBAAiB,YAAW,SAAS;CACrD,IAAI,CAAC,WACH,QAAQ;MACH;EACL,GAAG,OAAO,wBAAwB,SAAS;EAC3C,OAAO,WAAW,oBAAoB;EACtC,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAItC,MAAM,mBAAmB,UACtB,UACC,YACE,gBACA;GAAC,GAAG;GAAK,qCAAqC;EAAS,GACvD,QACF,GACA,cACA,mBACF,EACC,gBAAgB,eAAe,QAAQ,CAAC;EAC3C,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;CACF;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,cAAc;CAClB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAc,cAAc,oBAAoB,EAAE,WAAW;GAC3D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,YAAiB,WACT,WAAW,QAAQ,0BAA0B,SAC7C,WAAW,QAAQ,uBAAuB,CAClD;CAOA,MAAM,gBAAgB,qBAClB,cACA,QAAQ,IAAI,CAAC,aAAa,eAAe,CAAC;CAK9C,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,WAAW,QAAQ,eAAe;EAClC,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
|
|
@@ -10,6 +10,11 @@ export interface PurgeLock {
|
|
|
10
10
|
}
|
|
11
11
|
/** A PurgeLock that also constrains the replica that can be restored. */
|
|
12
12
|
export interface ConstrainingPurgeLock extends PurgeLock, ReplicaConstraints {
|
|
13
|
+
/**
|
|
14
|
+
* The head of the change log when the lock was acquired, from which (or
|
|
15
|
+
* from past which) the change log is resumed.
|
|
16
|
+
*/
|
|
17
|
+
readonly headWatermark: string;
|
|
13
18
|
}
|
|
14
19
|
/**
|
|
15
20
|
* Acquires a purge lock on the PG change-log (if it is not empty).
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replica-restore.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAEL,KAAK,kBAAkB,EAExB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gCAAgC,CAAC;AAC9D,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAEnD,MAAM,WAAW,SAAS;IACxB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,yEAAyE;AACzE,MAAM,WAAW,qBAAsB,SAAQ,SAAS,EAAE,kBAAkB;
|
|
1
|
+
{"version":3,"file":"replica-restore.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAEL,KAAK,kBAAkB,EAExB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gCAAgC,CAAC;AAC9D,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAEnD,MAAM,WAAW,SAAS;IACxB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,yEAAyE;AACzE,MAAM,WAAW,qBAAsB,SAAQ,SAAS,EAAE,kBAAkB;IAC1E;;;OAGG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,CACnC,aAAa,CAAC,EAAE,MAAM,KACnB,OAAO,CAAC,qBAAqB,GAAG,IAAI,GAAG,aAAa,CAAC,CAAC;AAE3D,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,GAAG,SAAS,CAAC;IACtD;;;;;OAKG;IACH,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,YAAY,EAAE,YAAY,CAAC;IAC3B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;;;;;;;;OASG;IACH,0BAA0B,EAAE,OAAO,CAAC;IAEpC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE5C;;;;;;;;;;;OAWG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAE1B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,CAAC;AAEhE;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,SAAS,CAAC;CACtB,CAAC;AAOF;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,WAAW,CAAC,EAAE,iBAAiB,GAAG,SAAS,EAC3C,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,GAChC,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAuDtC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replica-restore.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport type {LitestreamConfig} from '../../../config/normalize.ts';\nimport {\n tryRestore,\n type ReplicaConstraints,\n type RestoreResult,\n} from '../../litestream/commands.ts';\nimport {\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreRuns,\n} from '../../litestream/metrics.ts';\nimport type {WalKeeper} from '../../litestream/wal-keeper.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource} from '../change-source.ts';\nimport type {InitCleanup} from './init-cleanup.ts';\n\nexport interface PurgeLock {\n release(): Promise<void>;\n}\n\n/** A PurgeLock that also constrains the replica that can be restored. */\nexport interface ConstrainingPurgeLock extends PurgeLock, ReplicaConstraints {}\n\n/**\n * Acquires a purge lock on the PG change-log (if it is not empty).\n *\n * When resuming the change-log from a replication slot, `slotWatermark` is the\n * slot's position (as a watermark). If the change-log's head is behind it, the\n * change-log may be missing transactions that the slot will not stream, in\n * which case nothing is locked and `'behind-slot'` is returned.\n */\nexport type PgChangeLogPurgeLocker = (\n slotWatermark?: string,\n) => Promise<ConstrainingPurgeLock | null | 'behind-slot'>;\n\nexport type RestoreOptions = {\n litestream?: LitestreamConfig;\n /**\n * With the PG change-log enabled, purge-locks the change-log before the\n * replica is restored, which constrains the replica to one from which the\n * change-log can be resumed. The lock is acquired as late as possible (i.e.\n * after a replication slot is created, if applicable) because, if the\n * change-db is the upstream db, a transaction holding the lock would block\n * the creation of a replication slot.\n */\n acquirePurgeLock?: PgChangeLogPurgeLocker | undefined;\n /**\n * Resources acquired by the initialization (e.g. a claimed replication slot)\n * are registered here to be released if the initialization attempt fails.\n * Note that a purge lock acquired with `acquirePurgeLock` must be registered\n * here by the locker, so that it is released on failure.\n */\n cleanup?: InitCleanup | undefined;\n};\n\nexport type InitializeResult = {\n subscriptionState: SubscriptionState;\n changeSource: ChangeSource;\n destinationBackupURL: string | undefined;\n /**\n * The replica this change stream belongs to, from the upstream `replicas`\n * table. It is part of the identity the SQLite change log records, because a\n * generation (i.e. `replicaVersion`) is shared by every sibling of a forked\n * replica and so cannot distinguish two siblings' logs. `null` when the\n * change source has no upstream table to identify itself from.\n */\n replicaID: string | null;\n\n /**\n * Whether server readiness should be gated on the first litestream backup,\n * in order to ensure that a newly connecting subscriber can restore from\n * this change-streamer's backup.\n *\n * This should be true when the backup destination differs from where the\n * backup was restored from, which is the case for (1) initial-sync and\n * (2) most initialization cases in RMv2, with the exception being when\n * an abandoned replica is resumed.\n */\n waitForBackupBeforeServing: boolean;\n\n /**\n * Whether the PG change-log's head was behind the position of the\n * replication slot from which replication resumes (see\n * {@link PgChangeLogPurgeLocker}), in which case it is re-initialized from\n * the restored replica if it does not contain the replica's changes.\n */\n pgChangeLogBehindSlot?: boolean | undefined;\n\n /**\n * Whether the litestream backup to `destinationBackupURL` starts a new\n * (litestream v5) backup lineage, i.e. one that no local litestream state\n * belongs to. Any such state (e.g. from a previous run of a restarted\n * container, whose restore reused the existing replica) describes a\n * different lineage and must be discarded before replicating, or litestream\n * resolves its position from it and fails (and auto-recovers) on its first\n * sync to the empty lineage.\n *\n * This is false when the backup continues a lineage from the local\n * litestream state (see {@link ResumableBackup}).\n */\n newBackupLineage: boolean;\n\n /**\n * Set when the backup to `destinationBackupURL` continues from the local\n * litestream state (see {@link ResumableBackup}). It is registered with the\n * initialization's {@link InitCleanup}, which releases it if the attempt\n * fails; otherwise the caller must release it once the litestream backup\n * has started.\n */\n walKeeper?: WalKeeper | undefined;\n};\n\n/**\n * Where the backup goes after a restore, when it can continue from the\n * restored replica's local litestream state instead of starting a new lineage\n * with a full snapshot:\n * - `fork`: a new, unique subfolder of `baseURL`, which the restore seeds with\n * the restored backup files.\n */\nexport type BackupDestination = {type: 'fork'; baseURL: string};\n\n/**\n * A backup lineage at `backupPath` that the litestream backup continues from\n * the local litestream state, whose WAL `walKeeper` holds.\n */\nexport type ResumableBackup = {\n backupPath: string;\n walKeeper: WalKeeper;\n};\n\n// A short retry is much cheaper than an initial Postgres sync, while keeping a\n// persistent failure from delaying startup appreciably.\nconst MAX_RESTORE_ATTEMPTS = 3;\nconst RESTORE_RETRY_DELAY_MS = 5_000;\n\n/**\n * Restores the replica from `config.backupURL`. With a `destination`, it\n * returns the backup that continues from the restored replica, if the\n * restore prepared one (e.g. it was not skipped because the replica already\n * exists). Its WAL keeper is registered with `cleanup`.\n */\nexport async function restoreReplica(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints | undefined,\n destination?: BackupDestination | undefined,\n cleanup?: InitCleanup | undefined,\n): Promise<ResumableBackup | undefined> {\n const start = performance.now();\n let result: RestoreResult | undefined;\n try {\n // `tryRestore` returns ordinary restore outcomes (no backup or an invalid\n // replica) rather than throwing; preserve their existing immediate\n // initial-sync fallback. Retry every thrown restore error twice before\n // falling back to the initial Postgres sync.\n for (let attemptNum = 1; attemptNum <= MAX_RESTORE_ATTEMPTS; attemptNum++) {\n // Each attempt seeds a new subfolder, as a failed attempt can leave\n // files in its subfolder.\n const backupPath =\n destination?.type === 'fork' ? String(Date.now()) : undefined;\n try {\n const attempt = await tryRestore(\n lc,\n config,\n replicaFile,\n replicaConstraints,\n 'replication_manager',\n undefined,\n backupPath\n ? {\n url: new URL(backupPath, destination?.baseURL).toString(),\n cleanup,\n }\n : undefined,\n );\n result = attempt.result;\n return backupPath && attempt.walKeeper\n ? {backupPath, walKeeper: attempt.walKeeper}\n : undefined;\n } catch (e) {\n if (attemptNum === MAX_RESTORE_ATTEMPTS) {\n lc.error?.(\n `litestream restore failed after ${attemptNum} attempts; resyncing the replica`,\n e,\n );\n break;\n }\n\n lc.warn?.(\n `litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`,\n e,\n );\n await sleep(RESTORE_RETRY_DELAY_MS);\n }\n }\n return undefined;\n } finally {\n const attrs = litestreamRestoreMetricAttrs(config, 'replication_manager');\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n"],"mappings":";;;;
|
|
1
|
+
{"version":3,"file":"replica-restore.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport type {LitestreamConfig} from '../../../config/normalize.ts';\nimport {\n tryRestore,\n type ReplicaConstraints,\n type RestoreResult,\n} from '../../litestream/commands.ts';\nimport {\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreRuns,\n} from '../../litestream/metrics.ts';\nimport type {WalKeeper} from '../../litestream/wal-keeper.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource} from '../change-source.ts';\nimport type {InitCleanup} from './init-cleanup.ts';\n\nexport interface PurgeLock {\n release(): Promise<void>;\n}\n\n/** A PurgeLock that also constrains the replica that can be restored. */\nexport interface ConstrainingPurgeLock extends PurgeLock, ReplicaConstraints {\n /**\n * The head of the change log when the lock was acquired, from which (or\n * from past which) the change log is resumed.\n */\n readonly headWatermark: string;\n}\n\n/**\n * Acquires a purge lock on the PG change-log (if it is not empty).\n *\n * When resuming the change-log from a replication slot, `slotWatermark` is the\n * slot's position (as a watermark). If the change-log's head is behind it, the\n * change-log may be missing transactions that the slot will not stream, in\n * which case nothing is locked and `'behind-slot'` is returned.\n */\nexport type PgChangeLogPurgeLocker = (\n slotWatermark?: string,\n) => Promise<ConstrainingPurgeLock | null | 'behind-slot'>;\n\nexport type RestoreOptions = {\n litestream?: LitestreamConfig;\n /**\n * With the PG change-log enabled, purge-locks the change-log before the\n * replica is restored, which constrains the replica to one from which the\n * change-log can be resumed. The lock is acquired as late as possible (i.e.\n * after a replication slot is created, if applicable) because, if the\n * change-db is the upstream db, a transaction holding the lock would block\n * the creation of a replication slot.\n */\n acquirePurgeLock?: PgChangeLogPurgeLocker | undefined;\n /**\n * Resources acquired by the initialization (e.g. a claimed replication slot)\n * are registered here to be released if the initialization attempt fails.\n * Note that a purge lock acquired with `acquirePurgeLock` must be registered\n * here by the locker, so that it is released on failure.\n */\n cleanup?: InitCleanup | undefined;\n};\n\nexport type InitializeResult = {\n subscriptionState: SubscriptionState;\n changeSource: ChangeSource;\n destinationBackupURL: string | undefined;\n /**\n * The replica this change stream belongs to, from the upstream `replicas`\n * table. It is part of the identity the SQLite change log records, because a\n * generation (i.e. `replicaVersion`) is shared by every sibling of a forked\n * replica and so cannot distinguish two siblings' logs. `null` when the\n * change source has no upstream table to identify itself from.\n */\n replicaID: string | null;\n\n /**\n * Whether server readiness should be gated on the first litestream backup,\n * in order to ensure that a newly connecting subscriber can restore from\n * this change-streamer's backup.\n *\n * This should be true when the backup destination differs from where the\n * backup was restored from, which is the case for (1) initial-sync and\n * (2) most initialization cases in RMv2, with the exception being when\n * an abandoned replica is resumed.\n */\n waitForBackupBeforeServing: boolean;\n\n /**\n * Whether the PG change-log's head was behind the position of the\n * replication slot from which replication resumes (see\n * {@link PgChangeLogPurgeLocker}), in which case it is re-initialized from\n * the restored replica if it does not contain the replica's changes.\n */\n pgChangeLogBehindSlot?: boolean | undefined;\n\n /**\n * Whether the litestream backup to `destinationBackupURL` starts a new\n * (litestream v5) backup lineage, i.e. one that no local litestream state\n * belongs to. Any such state (e.g. from a previous run of a restarted\n * container, whose restore reused the existing replica) describes a\n * different lineage and must be discarded before replicating, or litestream\n * resolves its position from it and fails (and auto-recovers) on its first\n * sync to the empty lineage.\n *\n * This is false when the backup continues a lineage from the local\n * litestream state (see {@link ResumableBackup}).\n */\n newBackupLineage: boolean;\n\n /**\n * Set when the backup to `destinationBackupURL` continues from the local\n * litestream state (see {@link ResumableBackup}). It is registered with the\n * initialization's {@link InitCleanup}, which releases it if the attempt\n * fails; otherwise the caller must release it once the litestream backup\n * has started.\n */\n walKeeper?: WalKeeper | undefined;\n};\n\n/**\n * Where the backup goes after a restore, when it can continue from the\n * restored replica's local litestream state instead of starting a new lineage\n * with a full snapshot:\n * - `fork`: a new, unique subfolder of `baseURL`, which the restore seeds with\n * the restored backup files.\n */\nexport type BackupDestination = {type: 'fork'; baseURL: string};\n\n/**\n * A backup lineage at `backupPath` that the litestream backup continues from\n * the local litestream state, whose WAL `walKeeper` holds.\n */\nexport type ResumableBackup = {\n backupPath: string;\n walKeeper: WalKeeper;\n};\n\n// A short retry is much cheaper than an initial Postgres sync, while keeping a\n// persistent failure from delaying startup appreciably.\nconst MAX_RESTORE_ATTEMPTS = 3;\nconst RESTORE_RETRY_DELAY_MS = 5_000;\n\n/**\n * Restores the replica from `config.backupURL`. With a `destination`, it\n * returns the backup that continues from the restored replica, if the\n * restore prepared one (e.g. it was not skipped because the replica already\n * exists). Its WAL keeper is registered with `cleanup`.\n */\nexport async function restoreReplica(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints | undefined,\n destination?: BackupDestination | undefined,\n cleanup?: InitCleanup | undefined,\n): Promise<ResumableBackup | undefined> {\n const start = performance.now();\n let result: RestoreResult | undefined;\n try {\n // `tryRestore` returns ordinary restore outcomes (no backup or an invalid\n // replica) rather than throwing; preserve their existing immediate\n // initial-sync fallback. Retry every thrown restore error twice before\n // falling back to the initial Postgres sync.\n for (let attemptNum = 1; attemptNum <= MAX_RESTORE_ATTEMPTS; attemptNum++) {\n // Each attempt seeds a new subfolder, as a failed attempt can leave\n // files in its subfolder.\n const backupPath =\n destination?.type === 'fork' ? String(Date.now()) : undefined;\n try {\n const attempt = await tryRestore(\n lc,\n config,\n replicaFile,\n replicaConstraints,\n 'replication_manager',\n undefined,\n backupPath\n ? {\n url: new URL(backupPath, destination?.baseURL).toString(),\n cleanup,\n }\n : undefined,\n );\n result = attempt.result;\n return backupPath && attempt.walKeeper\n ? {backupPath, walKeeper: attempt.walKeeper}\n : undefined;\n } catch (e) {\n if (attemptNum === MAX_RESTORE_ATTEMPTS) {\n lc.error?.(\n `litestream restore failed after ${attemptNum} attempts; resyncing the replica`,\n e,\n );\n break;\n }\n\n lc.warn?.(\n `litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`,\n e,\n );\n await sleep(RESTORE_RETRY_DELAY_MS);\n }\n }\n return undefined;\n } finally {\n const attrs = litestreamRestoreMetricAttrs(config, 'replication_manager');\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n"],"mappings":";;;;AA4IA,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;;;;;;;AAQ/B,eAAsB,eACpB,IACA,QACA,aACA,oBACA,aACA,SACsC;CACtC,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;EAKF,KAAK,IAAI,aAAa,GAAG,cAAc,sBAAsB,cAAc;GAGzE,MAAM,aACJ,aAAa,SAAS,SAAS,OAAO,KAAK,IAAI,CAAC,IAAI,KAAA;GACtD,IAAI;IACF,MAAM,UAAU,MAAM,WACpB,IACA,QACA,aACA,oBACA,uBACA,KAAA,GACA,aACI;KACE,KAAK,IAAI,IAAI,YAAY,aAAa,OAAO,EAAE,SAAS;KACxD;IACF,IACA,KAAA,CACN;IACA,SAAS,QAAQ;IACjB,OAAO,cAAc,QAAQ,YACzB;KAAC;KAAY,WAAW,QAAQ;IAAS,IACzC,KAAA;GACN,SAAS,GAAG;IACV,IAAI,eAAe,sBAAsB;KACvC,GAAG,QACD,mCAAmC,WAAW,mCAC9C,CACF;KACA;IACF;IAEA,GAAG,OACD,8BAA8B,WAAW,uBAAuB,uBAAuB,cAAc,aAAa,EAAE,MAAM,qBAAqB,IAC/I,CACF;IACA,MAAM,MAAM,sBAAsB;GACpC;EACF;EACA;CACF,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,qBAChC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF"}
|
|
@@ -2,7 +2,7 @@ import type { Readable } from 'node:stream';
|
|
|
2
2
|
import type { LogContext } from '@rocicorp/logger';
|
|
3
3
|
import { type PostgresDB } from '../../../types/pg.ts';
|
|
4
4
|
import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
|
|
5
|
-
import { type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
|
|
5
|
+
import { type ConstrainingPurgeLock, type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
|
|
6
6
|
import { type InitialSyncOptions, type ReplicaOptions, type ServerContext } from './initial-sync.ts';
|
|
7
7
|
import { type LSN } from './lsn.ts';
|
|
8
8
|
import { type ReplicaState } from './schema/shard.ts';
|
|
@@ -16,6 +16,31 @@ export type InitializeOptions = ReplicaOptions & {
|
|
|
16
16
|
* stream.
|
|
17
17
|
*/
|
|
18
18
|
export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, restoreOptions?: RestoreOptions, { epoch, slotPerReplica, backupV5, inactiveReplicaGracePeriodMs, }?: InitializeOptions, streamInboundTimeoutMs?: number | undefined): Promise<InitializeResult>;
|
|
19
|
+
/**
|
|
20
|
+
* The watermark below which the replication stream cannot (correctly) be
|
|
21
|
+
* started, i.e. from which it would skip transactions that the change log
|
|
22
|
+
* does not contain. It reflects the order in which each mode stores a
|
|
23
|
+
* transaction and then acks it to the slot, such that a correctly initialized
|
|
24
|
+
* stream always starts at or past it:
|
|
25
|
+
*
|
|
26
|
+
* * After an initial sync, the replica's watermark, which is the position
|
|
27
|
+
* at which the new slot was created, and the change log is (re)initialized
|
|
28
|
+
* at it.
|
|
29
|
+
* * With a slot per replica (RMv1.5 and RMv2), the minimum of the restored
|
|
30
|
+
* replica's watermark and the slot's position. Transactions are only acked
|
|
31
|
+
* once backed up (so a skipped interval ends at or before the replica), and
|
|
32
|
+
* if the change log is behind the slot it is re-initialized from the
|
|
33
|
+
* replica. Note that the slot can be past the replica (acked for keepalives,
|
|
34
|
+
* with nothing skipped), and the replica past the change log (forwarded
|
|
35
|
+
* before being stored, and streamed again from the slot), but the stream
|
|
36
|
+
* never starts below both. (With RMv1.5's purge lock, the change log's head
|
|
37
|
+
* is at or past the slot, so it does not lower the minimum.)
|
|
38
|
+
* * With a shared slot (RMv1), the change log's head when it was purge
|
|
39
|
+
* locked, as the stream resumes from the head, which only advances. The
|
|
40
|
+
* slot is not a reference: its owner acks it until it is taken over. Without
|
|
41
|
+
* a purge lock (i.e. an empty change log), there is no floor.
|
|
42
|
+
*/
|
|
43
|
+
export declare function minStartWatermark(replicaWatermark: string, initialSynced: boolean, slotWatermark: string | undefined, purgeLock: ConstrainingPurgeLock | null): string | undefined;
|
|
19
44
|
export declare function getSourceAndDestinationReplicas(lc: LogContext, sql: PostgresDB, shard: ShardID, epoch: number, slotFailover: boolean, gracePeriodMs: number, pollIntervalMs?: number): Promise<{
|
|
20
45
|
restoreFrom: ReplicaState;
|
|
21
46
|
replicateTo: ReplicaState;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOxE,OAAO,
|
|
1
|
+
{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOxE,OAAO,EAEL,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EACrB,KAAK,cAAc,EAEpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAiC,KAAK,GAAG,EAAC,MAAM,UAAU,CAAC;AASlE,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,GAC7B,GAAE,iBAKF,EACD,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CA6I3B;AAkKD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,wBAAgB,iBAAiB,CAC/B,gBAAgB,EAAE,MAAM,EACxB,aAAa,EAAE,OAAO,EACtB,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,SAAS,EAAE,qBAAqB,GAAG,IAAI,GACtC,MAAM,GAAG,SAAS,CAQpB;AAkBD,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM,EACrB,cAAc,SAA2B,GACxC,OAAO,CACN;IACE,WAAW,EAAE,YAAY,CAAC;IAC1B,WAAW,EAAE,YAAY,CAAC;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,GAAG,CAAC;IACb;;;OAGG;IACH,WAAW,EAAE,QAAQ,CAAC;CACvB,GACD,SAAS,CACZ,CAiKA"}
|
|
@@ -32,7 +32,7 @@ async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbF
|
|
|
32
32
|
const db = await connectPgClient(lc, upstreamURI, "change-source-init");
|
|
33
33
|
try {
|
|
34
34
|
await ensureShardSchema(lc, db, shard, syncOptions.installPartialIndexTriggers);
|
|
35
|
-
const { replica: restoredReplica, purgeLock, pgChangeLogBehindSlot, resumableBackup } = slotPerReplica ? await forkOrResumeReplica(lc, db, shard, epoch, syncOptions.replicationSlotFailover ?? false, replicaDbFile, restoreOptions, inactiveReplicaGracePeriodMs) : await selectAndRestoreReplica(lc, db, shard, replicaDbFile, restoreOptions);
|
|
35
|
+
const { replica: restoredReplica, purgeLock, pgChangeLogBehindSlot, resumableBackup, slotWatermark } = slotPerReplica ? await forkOrResumeReplica(lc, db, shard, epoch, syncOptions.replicationSlotFailover ?? false, replicaDbFile, restoreOptions, inactiveReplicaGracePeriodMs) : await selectAndRestoreReplica(lc, db, shard, replicaDbFile, restoreOptions);
|
|
36
36
|
let initialSyncedReplica;
|
|
37
37
|
await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, async (log, tx) => {
|
|
38
38
|
await restoreOptions.cleanup?.release();
|
|
@@ -53,7 +53,7 @@ async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbF
|
|
|
53
53
|
changeSource: new PostgresChangeSource(lc, upstreamURI, shard, upstreamReplica, pgVersion, {
|
|
54
54
|
backupPath,
|
|
55
55
|
backupV5
|
|
56
|
-
}, context, lagReportIntervalMs, syncOptions.textCopy, streamInboundTimeoutMs),
|
|
56
|
+
}, context, lagReportIntervalMs, syncOptions.textCopy, streamInboundTimeoutMs, minStartWatermark(subscriptionState.watermark, initialSyncedReplica !== void 0, slotWatermark, purgeLock)),
|
|
57
57
|
destinationBackupURL: backupPath && restoreOptions.litestream?.backupURL ? new URL(backupPath, restoreOptions.litestream.backupURL).toString() : restoreOptions.litestream?.backupURL,
|
|
58
58
|
replicaID: upstreamReplica.id,
|
|
59
59
|
waitForBackupBeforeServing: initialSyncedReplica !== void 0 || backupPath !== (restoredReplica?.backupPath ?? null),
|
|
@@ -108,8 +108,9 @@ async function forkOrResumeReplica(lc, sql, shard, epoch, slotFailover, replicaF
|
|
|
108
108
|
purgeLock: null
|
|
109
109
|
};
|
|
110
110
|
const { restoreFrom, replicateTo, slotLSN, slotSession } = result;
|
|
111
|
+
const slotWatermark = toStateVersionString(slotLSN);
|
|
111
112
|
cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () => slotSession.destroy());
|
|
112
|
-
const lock = acquirePurgeLock ? await acquirePurgeLock(
|
|
113
|
+
const lock = acquirePurgeLock ? await acquirePurgeLock(slotWatermark) : null;
|
|
113
114
|
const pgChangeLogBehindSlot = lock === "behind-slot";
|
|
114
115
|
const purgeLock = lock === "behind-slot" ? null : lock;
|
|
115
116
|
const constraints = restoreConstraints(purgeLock, restoreFrom);
|
|
@@ -131,15 +132,46 @@ async function forkOrResumeReplica(lc, sql, shard, epoch, slotFailover, replicaF
|
|
|
131
132
|
}, replicaFile, constraints, litestream.forkBackup ? {
|
|
132
133
|
type: "fork",
|
|
133
134
|
baseURL: backupBaseURL
|
|
134
|
-
} : void 0, cleanup)
|
|
135
|
+
} : void 0, cleanup),
|
|
136
|
+
slotWatermark
|
|
135
137
|
};
|
|
136
138
|
}
|
|
137
139
|
return {
|
|
138
140
|
replica: replicateTo,
|
|
139
141
|
purgeLock,
|
|
140
|
-
pgChangeLogBehindSlot
|
|
142
|
+
pgChangeLogBehindSlot,
|
|
143
|
+
slotWatermark
|
|
141
144
|
};
|
|
142
145
|
}
|
|
146
|
+
/**
|
|
147
|
+
* The watermark below which the replication stream cannot (correctly) be
|
|
148
|
+
* started, i.e. from which it would skip transactions that the change log
|
|
149
|
+
* does not contain. It reflects the order in which each mode stores a
|
|
150
|
+
* transaction and then acks it to the slot, such that a correctly initialized
|
|
151
|
+
* stream always starts at or past it:
|
|
152
|
+
*
|
|
153
|
+
* * After an initial sync, the replica's watermark, which is the position
|
|
154
|
+
* at which the new slot was created, and the change log is (re)initialized
|
|
155
|
+
* at it.
|
|
156
|
+
* * With a slot per replica (RMv1.5 and RMv2), the minimum of the restored
|
|
157
|
+
* replica's watermark and the slot's position. Transactions are only acked
|
|
158
|
+
* once backed up (so a skipped interval ends at or before the replica), and
|
|
159
|
+
* if the change log is behind the slot it is re-initialized from the
|
|
160
|
+
* replica. Note that the slot can be past the replica (acked for keepalives,
|
|
161
|
+
* with nothing skipped), and the replica past the change log (forwarded
|
|
162
|
+
* before being stored, and streamed again from the slot), but the stream
|
|
163
|
+
* never starts below both. (With RMv1.5's purge lock, the change log's head
|
|
164
|
+
* is at or past the slot, so it does not lower the minimum.)
|
|
165
|
+
* * With a shared slot (RMv1), the change log's head when it was purge
|
|
166
|
+
* locked, as the stream resumes from the head, which only advances. The
|
|
167
|
+
* slot is not a reference: its owner acks it until it is taken over. Without
|
|
168
|
+
* a purge lock (i.e. an empty change log), there is no floor.
|
|
169
|
+
*/
|
|
170
|
+
function minStartWatermark(replicaWatermark, initialSynced, slotWatermark, purgeLock) {
|
|
171
|
+
if (initialSynced) return replicaWatermark;
|
|
172
|
+
if (slotWatermark !== void 0) return replicaWatermark < slotWatermark ? replicaWatermark : slotWatermark;
|
|
173
|
+
return purgeLock?.headWatermark;
|
|
174
|
+
}
|
|
143
175
|
function restoreConstraints(purgeLock, { generation }) {
|
|
144
176
|
return purgeLock ?? {
|
|
145
177
|
replicaVersion: generation,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport type {ReplicaConstraints} from '../../litestream/commands.ts';\nimport {\n getSubscriptionStateAndContext,\n type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n restoreReplica,\n type ConstrainingPurgeLock,\n type InitializeResult,\n type RestoreOptions,\n type ResumableBackup,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n initialSync,\n type InitialSyncOptions,\n type ReplicaOptions,\n type ServerContext,\n} from './initial-sync.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n claimSlotForResumption,\n createReplicaAndSlot,\n type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n dropShard,\n getActiveReplicas,\n getReplicaAtVersion,\n getRestoreCandidates,\n initRestoreReplica,\n internalPublicationPrefix,\n type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n // Create a new slot for the replica rather than taking over\n // an existing one (i.e. high-availability mode).\n slotPerReplica: boolean | undefined;\n\n inactiveReplicaGracePeriodMs: number;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n syncOptions: InitialSyncOptions,\n context: ServerContext,\n lagReportIntervalMs = 0,\n restoreOptions: RestoreOptions = {},\n {\n epoch,\n slotPerReplica,\n backupV5,\n inactiveReplicaGracePeriodMs,\n }: InitializeOptions = {\n epoch: 0,\n slotPerReplica: false,\n backupV5: true,\n inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n },\n streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n try {\n await ensureShardSchema(\n lc,\n db,\n shard,\n syncOptions.installPartialIndexTriggers,\n );\n\n const {\n replica: restoredReplica,\n purgeLock,\n pgChangeLogBehindSlot,\n resumableBackup,\n } = slotPerReplica\n ? await forkOrResumeReplica(\n lc,\n db,\n shard,\n epoch,\n syncOptions.replicationSlotFailover ?? false,\n replicaDbFile,\n restoreOptions,\n inactiveReplicaGracePeriodMs,\n )\n : await selectAndRestoreReplica(\n lc,\n db,\n shard,\n replicaDbFile,\n restoreOptions,\n );\n\n let initialSyncedReplica: ReplicaState | undefined;\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n async (log, tx) => {\n // Falling back to initial sync abandons the restore, so release the\n // resources acquired for it, e.g. a replication slot claimed (or\n // created) for a resumed (or forked) replica, which would otherwise\n // remain active indefinitely.\n await restoreOptions.cleanup?.release();\n // The purge lock on the change-db must be released before performing\n // initial sync; if the change-db and upstream are the same db, a lock-holding\n // transaction will prevent a replication slot from being created. This awkward\n // dependency can go away with RMv2. (The purge lock is also released by\n // the cleanup when it is registered with it; releasing it is idempotent.)\n await purgeLock?.release();\n initialSyncedReplica = await initialSync(\n log,\n shard,\n tx,\n upstreamURI,\n syncOptions,\n context,\n {epoch, backupV5},\n restoreOptions.cleanup,\n );\n },\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionStateAndContext(\n new StatementRunner(replica),\n );\n replica.close();\n\n // Check that upstream is properly setup, and throw an AutoReset to re-run\n // initial sync if not.\n const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n lc,\n db,\n shard,\n subscriptionState,\n (initialSyncedReplica ?? restoredReplica)?.id,\n );\n\n // With litestream v5, every replication-manager backs up to its own\n // backup lineage.\n const newBackupLineage = backupV5 && !resumableBackup;\n const backupPath = initialSyncedReplica\n ? // If initial sync was performed, use that initial backupPath.\n initialSyncedReplica.backupPath\n : // Otherwise, use the lineage prepared by the restore, or a new, unique\n // path when backing up with litestream v5. This will be recorded in the\n // replicas table by the PostgresChangeSource.\n backupV5\n ? (resumableBackup?.backupPath ?? String(Date.now()))\n : (restoredReplica?.backupPath ?? null);\n\n const changeSource = new PostgresChangeSource(\n lc,\n upstreamURI,\n shard,\n upstreamReplica,\n pgVersion,\n {backupPath, backupV5},\n context,\n lagReportIntervalMs,\n syncOptions.textCopy,\n streamInboundTimeoutMs,\n );\n\n const destinationBackupURL =\n backupPath && restoreOptions.litestream?.backupURL\n ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n restoreOptions.litestream?.backupURL;\n\n return {\n subscriptionState,\n changeSource,\n destinationBackupURL,\n // The replica this change stream belongs to. It is part of the identity\n // the SQLite change log records, because a generation (i.e.\n // `replicaVersion`) is shared by every sibling of a forked replica and so\n // cannot distinguish two siblings' logs.\n replicaID: upstreamReplica.id,\n waitForBackupBeforeServing:\n // Wait for the first backup if there was an initial sync,\n initialSyncedReplica !== undefined ||\n // or if the destination differs from where it was restored\n // (i.e. backupV5).\n backupPath !== (restoredReplica?.backupPath ?? null),\n pgChangeLogBehindSlot,\n newBackupLineage,\n walKeeper: resumableBackup?.walKeeper,\n };\n } finally {\n await db.end();\n }\n}\n\ntype RestoredReplica = {\n replica: ReplicaState | undefined;\n purgeLock: ConstrainingPurgeLock | null;\n pgChangeLogBehindSlot?: boolean | undefined;\n resumableBackup?: ResumableBackup | undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n// intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n replicaFile: string,\n {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n): Promise<RestoredReplica> {\n // The purge lock constrains the generation of the replica to restore. With\n // a shared replication slot, the change-log is resumed from its head\n // without being checked against the slot's position: the slot is still\n // being acked by the previous replication-manager, on an idle upstream past\n // the change-log's head (see PgChangeLogPurgeLocker).\n const lock = await acquirePurgeLock?.();\n assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n const purgeLock = lock ?? null;\n const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n // filter to the generation specified by the purge lock, if present\n ({generation}) => generation === (purgeLock?.replicaVersion ?? generation),\n );\n if (replicas.length === 0) {\n lc.info?.(`no suitable replicas to restore from`, {replicas});\n return {replica: undefined, purgeLock};\n }\n const [replica] = replicas;\n\n if (litestream?.backupURL) {\n const constraints = restoreConstraints(purgeLock, replica);\n const {backupURL: backupBaseURL} = litestream;\n const {slot, backupPath, confirmedFlushLsn} = replica;\n const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n lc.info?.(\n `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n {replicas},\n );\n const resumableBackup = await restoreReplica(\n lc,\n {...litestream, backupURL}, // includes the replica's backup sub-path\n replicaFile,\n constraints,\n litestream.forkBackup\n ? {type: 'fork', baseURL: backupBaseURL}\n : undefined,\n cleanup,\n );\n return {replica, purgeLock, resumableBackup};\n }\n return {replica, purgeLock};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n epoch: number,\n slotFailover: boolean,\n replicaFile: string,\n {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n gracePeriodMs: number,\n): Promise<RestoredReplica> {\n const result = await getSourceAndDestinationReplicas(\n lc,\n sql,\n shard,\n epoch,\n slotFailover,\n gracePeriodMs,\n );\n if (!result) {\n // can't restore, must initial-sync\n return {replica: undefined, purgeLock: null};\n }\n const {restoreFrom, replicateTo, slotLSN, slotSession} = result;\n // The slot is held active by a session in this process until it is taken\n // over by the PostgresChangeSource. If initialization fails before then,\n // the session must be released, or the slot remains active indefinitely,\n // preventing the replica from being resumed (or forked) by a retry.\n cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () =>\n slotSession.destroy(),\n );\n\n // Acquired only after the replication slot has been created (or claimed),\n // and only if the change-log can be resumed from the slot's position.\n //\n // For a fork, this is the case whenever the change-log is maintained (i.e.\n // by a replication-manager that acks the slot only once a transaction has\n // been stored in the change-log): the fork waits for the source replica's\n // slot to pass the position after the creation of the new replica, which is\n // a transaction (in the publication) after the new slot's position.\n //\n // For a resumed slot, the slot can also have been acked past the head of a\n // maintained change-log, for changes outside of the publication (e.g. on an\n // idle upstream). The change-log is then re-initialized unnecessarily, which\n // resets subscribers behind the replica. (This could be avoided by recording\n // the positions acked for such changes in the change-log, e.g. an\n // `ackedThrough` column, and checking the slot's position against that.\n // Resumption is uncommon, however, and involves downtime anyway.)\n const lock = acquirePurgeLock\n ? await acquirePurgeLock(toStateVersionString(slotLSN))\n : null;\n const pgChangeLogBehindSlot = lock === 'behind-slot';\n const purgeLock = lock === 'behind-slot' ? null : lock;\n // Absent a purge lock, still constrain the restore to the source's\n // generation, so that a local replica left behind by a different\n // generation (e.g. a failed initial sync) is not mistaken for it.\n const constraints = restoreConstraints(purgeLock, restoreFrom);\n\n if (litestream?.backupURL) {\n const {backupURL: backupBaseURL} = litestream;\n const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n lc.info?.(\n `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n {restoreFrom, replicateTo},\n );\n const resumableBackup = await restoreReplica(\n lc,\n {...litestream, backupURL}, // includes the replica's backup sub-path\n replicaFile,\n constraints,\n litestream.forkBackup\n ? {type: 'fork', baseURL: backupBaseURL}\n : undefined,\n cleanup,\n );\n return {\n replica: replicateTo,\n purgeLock,\n pgChangeLogBehindSlot,\n resumableBackup,\n };\n }\n return {replica: replicateTo, purgeLock, pgChangeLogBehindSlot};\n}\n\nfunction restoreConstraints(\n purgeLock: ConstrainingPurgeLock | null,\n {generation}: ReplicaState,\n): ReplicaConstraints {\n return purgeLock ?? {replicaVersion: generation, minWatermark: ''};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 5_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n epoch: number,\n slotFailover: boolean,\n gracePeriodMs: number,\n pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n | {\n restoreFrom: ReplicaState;\n replicateTo: ReplicaState;\n /** The position of the slot from which replication resumes. */\n slotLSN: LSN;\n /**\n * The session holding the destination slot active until it is taken\n * over by the PostgresChangeSource. Destroying it releases the slot.\n */\n slotSession: Readable;\n }\n | undefined\n> {\n const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n let destination: ReplicationSlotResult<void> | undefined;\n // The position after the creation of the destination replica.\n let forkLSN: LSN | undefined;\n\n try {\n for (let i = 0; ; i++) {\n if (i > 0) {\n await sleep(pollIntervalMs);\n }\n const replicas = await getRestoreCandidates(lc, sql, shard, epoch);\n if (replicas.length === 0) {\n lc.info?.(`no suitable replicas to restore from`, {replicas});\n destination?.initialSession.destroy();\n return undefined;\n }\n for (const replica of replicas) {\n // Track inactivity to resume replicas after a grace period.\n if (replica.active) {\n inactiveSince.delete(replica.id);\n } else if (!inactiveSince.has(replica.id)) {\n inactiveSince.set(replica.id, Date.now());\n }\n\n // Note: Only `active` InitialSync replicas are returned from\n // getRestoreCandidates().\n if (replica.stage === InitialSync) {\n // Log periodically; initial-sync can be long\n if (i % 12 === 0) {\n lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n }\n break;\n }\n\n if (replica.stage === Replicate) {\n const {active, confirmedFlushLsn} = replica;\n if (active) {\n // Create a replication slot to fork the active replica.\n destination ??= await createReplicaAndSlot(\n lc,\n sql,\n 'fork-replica-session',\n shard,\n epoch,\n Date.now().toString(), // replicaID\n slotFailover,\n {\n backupPath: null, // set only after the backup has been confirmed\n backupV5: true, // RMv2 requires backupV5\n },\n () => promiseVoid,\n ReplicaStage.Restore,\n );\n // The replica row of the destination is inserted in a transaction\n // (in the publication) after the destination slot's position.\n // Waiting for the source replica's slot to pass it (rather than\n // just the destination slot's position) ensures that a source\n // replica that maintains the PG change-log has stored a\n // transaction past the destination slot's position, i.e. that the\n // change-log can be resumed from the destination slot.\n forkLSN ??= await currentWalLSN(sql);\n if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n lc.info?.(\n `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n {replica},\n );\n break;\n }\n }\n\n // If the replica is past the destination LSN, fork it, regardless of\n // whether its slot is still active. Forking is preferable to resuming,\n // as resuming removes the replica from being a candidate for\n // subsequent forks, and carries the risk of stealing the slot from a\n // task attempting to reconnect.\n if (\n destination &&\n forkLSN &&\n toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n ) {\n lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n replica,\n });\n const replicateTo = must(\n await initRestoreReplica(sql, shard, {\n sourceID: replica.id,\n destID: destination.replica.id,\n }),\n `replica ${destination.replica.id} was deleted`,\n );\n return {\n restoreFrom: replica,\n replicateTo,\n slotLSN: destination.slot.consistent_point,\n slotSession: destination.initialSession,\n };\n }\n }\n\n if (\n replica.stage === ReplicaStage.Restore &&\n replica.active &&\n replica.id !== destination?.replica.id\n ) {\n // Another task is restoring (e.g. resuming) this replica.\n if (i % 12 === 0) {\n lc.info?.(\n `waiting for replica ${replica.id}@${replica.slot} being restored by another task`,\n {replica},\n );\n }\n }\n\n if (replica.backupPath && !replica.active) {\n // An inactive replica that has a backupPath may be orphaned, or it\n // may be an active task that was temporarily disconnected and\n // attempting to reestablish a session.\n //\n // Resume the replica after a grace period to avoid stealing the slot\n // from an active task that was temporarily disconnected. Resumption\n // is a last resort only taken in the absence of alternatives.\n const now = Date.now();\n const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n lc.info?.(\n `replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`,\n {replica},\n );\n if (inactiveMs >= gracePeriodMs) {\n const reserved = await claimSlotForResumption(\n lc,\n sql,\n shard,\n 'resume-replica',\n replica,\n );\n if (!reserved) {\n lc.warn?.(\n `unable to resume replica ${replica.id}@${replica.slot}`,\n );\n } else {\n lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n // If a new replication slot was created in anticipation of forking,\n // cancel it\n destination?.initialSession.destroy();\n return {\n restoreFrom: replica,\n replicateTo: reserved.replica,\n // The slot is inactive, so its position is fixed.\n slotLSN: replica.confirmedFlushLsn,\n slotSession: reserved.reservation,\n };\n }\n }\n }\n }\n }\n } catch (e) {\n destination?.initialSession.destroy();\n throw e;\n }\n}\n\nasync function checkAndUpdateUpstream(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n {\n replicaVersion,\n publications: subscribed,\n initialSyncContext,\n }: SubscriptionStateAndContext,\n replicaID: string | undefined,\n) {\n const upstreamReplica = await getReplicaAtVersion(\n lc,\n sql,\n shard,\n replicaVersion,\n replicaID,\n initialSyncContext,\n );\n if (!upstreamReplica) {\n throw new AutoResetSignal(\n `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n );\n }\n\n // Verify that the publications match what is being replicated.\n const requested = shard.publications.toSorted();\n const replicated = upstreamReplica.publications\n .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n .sort();\n if (!deepEqual(requested, replicated)) {\n lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n throw new AutoResetSignal(\n `Requested publications [${requested}] do not match configured ` +\n `publications: [${replicated}]`,\n );\n }\n\n // Sanity check: The subscription state on the replica should have the\n // same publications. This should be guaranteed by the equivalence of the\n // replicaVersion, but it doesn't hurt to verify.\n if (!deepEqual(upstreamReplica.publications, subscribed)) {\n throw new AutoResetSignal(\n `Upstream publications [${upstreamReplica.publications}] do not ` +\n `match subscribed publications [${subscribed}]`,\n );\n }\n\n // Verify that the publications exist.\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n `.values();\n if (exists.length !== subscribed.length) {\n throw new AutoResetSignal(\n `Upstream publications [${exists.flat()}] do not contain ` +\n `all subscribed publications [${subscribed}]`,\n );\n }\n\n const {slot} = upstreamReplica;\n const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n /*sql*/ `\n SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n WHERE slot_name = ${slot}`;\n if (result.length === 0) {\n throw new AutoResetSignal(`replication slot ${slot} is missing`);\n }\n const [{restartLSN, walStatus}] = result;\n if (restartLSN === null || walStatus === 'lost') {\n throw new AutoResetSignal(\n `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n );\n }\n const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgEA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,iCACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;AAChC,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,uBACA,oBACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,MAAM,eAAe,SAAS,QAAQ;GAMtC,MAAM,WAAW,QAAQ;GACzB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;GAAQ,GAChB,eAAe,OACjB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAIA,MAAM,mBAAmB,YAAY,CAAC;EACtC,MAAM,aAAa,uBAEf,qBAAqB,aAIrB,WACG,iBAAiB,cAAc,OAAO,KAAK,IAAI,CAAC,IAChD,iBAAiB,cAAc;EAqBtC,OAAO;GACL;GACA,cAAA,IArBuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;GAAQ,GACrB,SACA,qBACA,YAAY,UACZ,sBAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;GACA;GACA,WAAW,iBAAiB;EAC9B;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAWA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,kBAAkB,WACL;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBAAgB,gBAAgB,WAAW,kBAAkB,WACjE;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,cAAc,mBAAmB,WAAW,OAAO;EACzD,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EAWA,OAAO;GAAC;GAAS;GAAW,iBAAA,MAVE,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,GACJ,OACF;EAC2C;CAC7C;CACA,OAAO;EAAC;EAAS;CAAS;AAC5B;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,kBAAkB,WAC/B,eAC0B;CAC1B,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,SAAS,gBAAe;CAKzD,SAAS,UAAU,oBAAoB,YAAY,cACjD,YAAY,QAAQ,CACtB;CAkBA,MAAM,OAAO,mBACT,MAAM,iBAAiB,qBAAqB,OAAO,CAAC,IACpD;CACJ,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAIlD,MAAM,cAAc,mBAAmB,WAAW,WAAW;CAE7D,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EAWA,OAAO;GACL,SAAS;GACT;GACA;GACA,iBAAA,MAd4B,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,GACJ,OACF;EAMA;CACF;CACA,OAAO;EAAC,SAAS;EAAa;EAAW;CAAqB;AAChE;AAEA,SAAS,mBACP,WACA,EAAC,cACmB;CACpB,OAAO,aAAa;EAAC,gBAAgB;EAAY,cAAc;CAAE;AACnE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AACjC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BAcjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;CAEJ,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,IAAI,KAAK,OAAO,KAAK;GACjE,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,aAAa,eAAe,QAAQ;IACpC;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IAAI,IAAI,OAAO,GACb,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;MACZ,SACM,aACN,CACF;MAQA,YAAY,MAAM,cAAc,GAAG;MACnC,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OACA;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;OAC1B,aAAa,YAAY;MAC3B;KACF;IACF;IAEA,IACE,QAAQ,UAAU,KAClB,QAAQ,UACR,QAAQ,OAAO,aAAa,QAAQ;SAGhC,IAAI,OAAO,GACb,GAAG,OACD,uBAAuB,QAAQ,GAAG,GAAG,QAAQ,KAAK,kCAClD,EAAC,QAAO,CACV;IAAA;IAIJ,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,wBAAwB,WAAW,KACzE,EAAC,QAAO,CACV;KACA,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,OACF;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,aAAa,eAAe,QAAQ;OACpC,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;QACjB,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,aAAa,eAAe,QAAQ;EACpC,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}
|
|
1
|
+
{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport type {ReplicaConstraints} from '../../litestream/commands.ts';\nimport {\n getSubscriptionStateAndContext,\n type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n restoreReplica,\n type ConstrainingPurgeLock,\n type InitializeResult,\n type RestoreOptions,\n type ResumableBackup,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n initialSync,\n type InitialSyncOptions,\n type ReplicaOptions,\n type ServerContext,\n} from './initial-sync.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n claimSlotForResumption,\n createReplicaAndSlot,\n type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n dropShard,\n getActiveReplicas,\n getReplicaAtVersion,\n getRestoreCandidates,\n initRestoreReplica,\n internalPublicationPrefix,\n type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n // Create a new slot for the replica rather than taking over\n // an existing one (i.e. high-availability mode).\n slotPerReplica: boolean | undefined;\n\n inactiveReplicaGracePeriodMs: number;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n syncOptions: InitialSyncOptions,\n context: ServerContext,\n lagReportIntervalMs = 0,\n restoreOptions: RestoreOptions = {},\n {\n epoch,\n slotPerReplica,\n backupV5,\n inactiveReplicaGracePeriodMs,\n }: InitializeOptions = {\n epoch: 0,\n slotPerReplica: false,\n backupV5: true,\n inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n },\n streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n try {\n await ensureShardSchema(\n lc,\n db,\n shard,\n syncOptions.installPartialIndexTriggers,\n );\n\n const {\n replica: restoredReplica,\n purgeLock,\n pgChangeLogBehindSlot,\n resumableBackup,\n slotWatermark,\n } = slotPerReplica\n ? await forkOrResumeReplica(\n lc,\n db,\n shard,\n epoch,\n syncOptions.replicationSlotFailover ?? false,\n replicaDbFile,\n restoreOptions,\n inactiveReplicaGracePeriodMs,\n )\n : await selectAndRestoreReplica(\n lc,\n db,\n shard,\n replicaDbFile,\n restoreOptions,\n );\n\n let initialSyncedReplica: ReplicaState | undefined;\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n async (log, tx) => {\n // Falling back to initial sync abandons the restore, so release the\n // resources acquired for it, e.g. a replication slot claimed (or\n // created) for a resumed (or forked) replica, which would otherwise\n // remain active indefinitely.\n await restoreOptions.cleanup?.release();\n // The purge lock on the change-db must be released before performing\n // initial sync; if the change-db and upstream are the same db, a lock-holding\n // transaction will prevent a replication slot from being created. This awkward\n // dependency can go away with RMv2. (The purge lock is also released by\n // the cleanup when it is registered with it; releasing it is idempotent.)\n await purgeLock?.release();\n initialSyncedReplica = await initialSync(\n log,\n shard,\n tx,\n upstreamURI,\n syncOptions,\n context,\n {epoch, backupV5},\n restoreOptions.cleanup,\n );\n },\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionStateAndContext(\n new StatementRunner(replica),\n );\n replica.close();\n\n // Check that upstream is properly setup, and throw an AutoReset to re-run\n // initial sync if not.\n const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n lc,\n db,\n shard,\n subscriptionState,\n (initialSyncedReplica ?? restoredReplica)?.id,\n );\n\n // With litestream v5, every replication-manager backs up to its own\n // backup lineage.\n const newBackupLineage = backupV5 && !resumableBackup;\n const backupPath = initialSyncedReplica\n ? // If initial sync was performed, use that initial backupPath.\n initialSyncedReplica.backupPath\n : // Otherwise, use the lineage prepared by the restore, or a new, unique\n // path when backing up with litestream v5. This will be recorded in the\n // replicas table by the PostgresChangeSource.\n backupV5\n ? (resumableBackup?.backupPath ?? String(Date.now()))\n : (restoredReplica?.backupPath ?? null);\n\n const changeSource = new PostgresChangeSource(\n lc,\n upstreamURI,\n shard,\n upstreamReplica,\n pgVersion,\n {backupPath, backupV5},\n context,\n lagReportIntervalMs,\n syncOptions.textCopy,\n streamInboundTimeoutMs,\n minStartWatermark(\n subscriptionState.watermark,\n initialSyncedReplica !== undefined,\n slotWatermark,\n purgeLock,\n ),\n );\n\n const destinationBackupURL =\n backupPath && restoreOptions.litestream?.backupURL\n ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n restoreOptions.litestream?.backupURL;\n\n return {\n subscriptionState,\n changeSource,\n destinationBackupURL,\n // The replica this change stream belongs to. It is part of the identity\n // the SQLite change log records, because a generation (i.e.\n // `replicaVersion`) is shared by every sibling of a forked replica and so\n // cannot distinguish two siblings' logs.\n replicaID: upstreamReplica.id,\n waitForBackupBeforeServing:\n // Wait for the first backup if there was an initial sync,\n initialSyncedReplica !== undefined ||\n // or if the destination differs from where it was restored\n // (i.e. backupV5).\n backupPath !== (restoredReplica?.backupPath ?? null),\n pgChangeLogBehindSlot,\n newBackupLineage,\n walKeeper: resumableBackup?.walKeeper,\n };\n } finally {\n await db.end();\n }\n}\n\ntype RestoredReplica = {\n replica: ReplicaState | undefined;\n purgeLock: ConstrainingPurgeLock | null;\n pgChangeLogBehindSlot?: boolean | undefined;\n resumableBackup?: ResumableBackup | undefined;\n /**\n * The position (as a watermark) of the replication slot from which\n * replication resumes, when it was created (for a fork) or claimed (for a\n * resumption), i.e. while it is held by this process. Absent for a shared\n * slot (RMv1), whose position is advanced by its current owner until it is\n * taken over.\n */\n slotWatermark?: string | undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n// intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n replicaFile: string,\n {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n): Promise<RestoredReplica> {\n // The purge lock constrains the generation of the replica to restore. With\n // a shared replication slot, the change-log is resumed from its head\n // without being checked against the slot's position: the slot is still\n // being acked by the previous replication-manager, on an idle upstream past\n // the change-log's head (see PgChangeLogPurgeLocker).\n const lock = await acquirePurgeLock?.();\n assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n const purgeLock = lock ?? null;\n const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n // filter to the generation specified by the purge lock, if present\n ({generation}) => generation === (purgeLock?.replicaVersion ?? generation),\n );\n if (replicas.length === 0) {\n lc.info?.(`no suitable replicas to restore from`, {replicas});\n return {replica: undefined, purgeLock};\n }\n const [replica] = replicas;\n\n if (litestream?.backupURL) {\n const constraints = restoreConstraints(purgeLock, replica);\n const {backupURL: backupBaseURL} = litestream;\n const {slot, backupPath, confirmedFlushLsn} = replica;\n const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n lc.info?.(\n `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n {replicas},\n );\n const resumableBackup = await restoreReplica(\n lc,\n {...litestream, backupURL}, // includes the replica's backup sub-path\n replicaFile,\n constraints,\n litestream.forkBackup\n ? {type: 'fork', baseURL: backupBaseURL}\n : undefined,\n cleanup,\n );\n return {replica, purgeLock, resumableBackup};\n }\n return {replica, purgeLock};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n epoch: number,\n slotFailover: boolean,\n replicaFile: string,\n {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n gracePeriodMs: number,\n): Promise<RestoredReplica> {\n const result = await getSourceAndDestinationReplicas(\n lc,\n sql,\n shard,\n epoch,\n slotFailover,\n gracePeriodMs,\n );\n if (!result) {\n // can't restore, must initial-sync\n return {replica: undefined, purgeLock: null};\n }\n const {restoreFrom, replicateTo, slotLSN, slotSession} = result;\n const slotWatermark = toStateVersionString(slotLSN);\n // The slot is held active by a session in this process until it is taken\n // over by the PostgresChangeSource. If initialization fails before then,\n // the session must be released, or the slot remains active indefinitely,\n // preventing the replica from being resumed (or forked) by a retry.\n cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () =>\n slotSession.destroy(),\n );\n\n // Acquired only after the replication slot has been created (or claimed),\n // and only if the change-log can be resumed from the slot's position.\n //\n // For a fork, this is the case whenever the change-log is maintained (i.e.\n // by a replication-manager that acks the slot only once a transaction has\n // been stored in the change-log): the fork waits for the source replica's\n // slot to pass the position after the creation of the new replica, which is\n // a transaction (in the publication) after the new slot's position.\n //\n // For a resumed slot, the slot can also have been acked past the head of a\n // maintained change-log, for changes outside of the publication (e.g. on an\n // idle upstream). The change-log is then re-initialized unnecessarily, which\n // resets subscribers behind the replica. (This could be avoided by recording\n // the positions acked for such changes in the change-log, e.g. an\n // `ackedThrough` column, and checking the slot's position against that.\n // Resumption is uncommon, however, and involves downtime anyway.)\n const lock = acquirePurgeLock ? await acquirePurgeLock(slotWatermark) : null;\n const pgChangeLogBehindSlot = lock === 'behind-slot';\n const purgeLock = lock === 'behind-slot' ? null : lock;\n // Absent a purge lock, still constrain the restore to the source's\n // generation, so that a local replica left behind by a different\n // generation (e.g. a failed initial sync) is not mistaken for it.\n const constraints = restoreConstraints(purgeLock, restoreFrom);\n\n if (litestream?.backupURL) {\n const {backupURL: backupBaseURL} = litestream;\n const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n lc.info?.(\n `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n {restoreFrom, replicateTo},\n );\n const resumableBackup = await restoreReplica(\n lc,\n {...litestream, backupURL}, // includes the replica's backup sub-path\n replicaFile,\n constraints,\n litestream.forkBackup\n ? {type: 'fork', baseURL: backupBaseURL}\n : undefined,\n cleanup,\n );\n return {\n replica: replicateTo,\n purgeLock,\n pgChangeLogBehindSlot,\n resumableBackup,\n slotWatermark,\n };\n }\n return {\n replica: replicateTo,\n purgeLock,\n pgChangeLogBehindSlot,\n slotWatermark,\n };\n}\n\n/**\n * The watermark below which the replication stream cannot (correctly) be\n * started, i.e. from which it would skip transactions that the change log\n * does not contain. It reflects the order in which each mode stores a\n * transaction and then acks it to the slot, such that a correctly initialized\n * stream always starts at or past it:\n *\n * * After an initial sync, the replica's watermark, which is the position\n * at which the new slot was created, and the change log is (re)initialized\n * at it.\n * * With a slot per replica (RMv1.5 and RMv2), the minimum of the restored\n * replica's watermark and the slot's position. Transactions are only acked\n * once backed up (so a skipped interval ends at or before the replica), and\n * if the change log is behind the slot it is re-initialized from the\n * replica. Note that the slot can be past the replica (acked for keepalives,\n * with nothing skipped), and the replica past the change log (forwarded\n * before being stored, and streamed again from the slot), but the stream\n * never starts below both. (With RMv1.5's purge lock, the change log's head\n * is at or past the slot, so it does not lower the minimum.)\n * * With a shared slot (RMv1), the change log's head when it was purge\n * locked, as the stream resumes from the head, which only advances. The\n * slot is not a reference: its owner acks it until it is taken over. Without\n * a purge lock (i.e. an empty change log), there is no floor.\n */\n// Exported for testing.\nexport function minStartWatermark(\n replicaWatermark: string,\n initialSynced: boolean,\n slotWatermark: string | undefined,\n purgeLock: ConstrainingPurgeLock | null,\n): string | undefined {\n if (initialSynced) {\n return replicaWatermark;\n }\n if (slotWatermark !== undefined) {\n return replicaWatermark < slotWatermark ? replicaWatermark : slotWatermark;\n }\n return purgeLock?.headWatermark;\n}\n\nfunction restoreConstraints(\n purgeLock: ConstrainingPurgeLock | null,\n {generation}: ReplicaState,\n): ReplicaConstraints {\n return purgeLock ?? {replicaVersion: generation, minWatermark: ''};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 5_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n epoch: number,\n slotFailover: boolean,\n gracePeriodMs: number,\n pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n | {\n restoreFrom: ReplicaState;\n replicateTo: ReplicaState;\n /** The position of the slot from which replication resumes. */\n slotLSN: LSN;\n /**\n * The session holding the destination slot active until it is taken\n * over by the PostgresChangeSource. Destroying it releases the slot.\n */\n slotSession: Readable;\n }\n | undefined\n> {\n const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n let destination: ReplicationSlotResult<void> | undefined;\n // The position after the creation of the destination replica.\n let forkLSN: LSN | undefined;\n\n try {\n for (let i = 0; ; i++) {\n if (i > 0) {\n await sleep(pollIntervalMs);\n }\n const replicas = await getRestoreCandidates(lc, sql, shard, epoch);\n if (replicas.length === 0) {\n lc.info?.(`no suitable replicas to restore from`, {replicas});\n destination?.initialSession.destroy();\n return undefined;\n }\n for (const replica of replicas) {\n // Track inactivity to resume replicas after a grace period.\n if (replica.active) {\n inactiveSince.delete(replica.id);\n } else if (!inactiveSince.has(replica.id)) {\n inactiveSince.set(replica.id, Date.now());\n }\n\n // Note: Only `active` InitialSync replicas are returned from\n // getRestoreCandidates().\n if (replica.stage === InitialSync) {\n // Log periodically; initial-sync can be long\n if (i % 12 === 0) {\n lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n }\n break;\n }\n\n if (replica.stage === Replicate) {\n const {active, confirmedFlushLsn} = replica;\n if (active) {\n // Create a replication slot to fork the active replica.\n destination ??= await createReplicaAndSlot(\n lc,\n sql,\n 'fork-replica-session',\n shard,\n epoch,\n Date.now().toString(), // replicaID\n slotFailover,\n {\n backupPath: null, // set only after the backup has been confirmed\n backupV5: true, // RMv2 requires backupV5\n },\n () => promiseVoid,\n ReplicaStage.Restore,\n );\n // The replica row of the destination is inserted in a transaction\n // (in the publication) after the destination slot's position.\n // Waiting for the source replica's slot to pass it (rather than\n // just the destination slot's position) ensures that a source\n // replica that maintains the PG change-log has stored a\n // transaction past the destination slot's position, i.e. that the\n // change-log can be resumed from the destination slot.\n forkLSN ??= await currentWalLSN(sql);\n if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n lc.info?.(\n `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n {replica},\n );\n break;\n }\n }\n\n // If the replica is past the destination LSN, fork it, regardless of\n // whether its slot is still active. Forking is preferable to resuming,\n // as resuming removes the replica from being a candidate for\n // subsequent forks, and carries the risk of stealing the slot from a\n // task attempting to reconnect.\n if (\n destination &&\n forkLSN &&\n toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n ) {\n lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n replica,\n });\n const replicateTo = must(\n await initRestoreReplica(sql, shard, {\n sourceID: replica.id,\n destID: destination.replica.id,\n }),\n `replica ${destination.replica.id} was deleted`,\n );\n return {\n restoreFrom: replica,\n replicateTo,\n slotLSN: destination.slot.consistent_point,\n slotSession: destination.initialSession,\n };\n }\n }\n\n if (\n replica.stage === ReplicaStage.Restore &&\n replica.active &&\n replica.id !== destination?.replica.id\n ) {\n // Another task is restoring (e.g. resuming) this replica.\n if (i % 12 === 0) {\n lc.info?.(\n `waiting for replica ${replica.id}@${replica.slot} being restored by another task`,\n {replica},\n );\n }\n }\n\n if (replica.backupPath && !replica.active) {\n // An inactive replica that has a backupPath may be orphaned, or it\n // may be an active task that was temporarily disconnected and\n // attempting to reestablish a session.\n //\n // Resume the replica after a grace period to avoid stealing the slot\n // from an active task that was temporarily disconnected. Resumption\n // is a last resort only taken in the absence of alternatives.\n const now = Date.now();\n const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n lc.info?.(\n `replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`,\n {replica},\n );\n if (inactiveMs >= gracePeriodMs) {\n const reserved = await claimSlotForResumption(\n lc,\n sql,\n shard,\n 'resume-replica',\n replica,\n );\n if (!reserved) {\n lc.warn?.(\n `unable to resume replica ${replica.id}@${replica.slot}`,\n );\n } else {\n lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n // If a new replication slot was created in anticipation of forking,\n // cancel it\n destination?.initialSession.destroy();\n return {\n restoreFrom: replica,\n replicateTo: reserved.replica,\n // The slot is inactive, so its position is fixed.\n slotLSN: replica.confirmedFlushLsn,\n slotSession: reserved.reservation,\n };\n }\n }\n }\n }\n }\n } catch (e) {\n destination?.initialSession.destroy();\n throw e;\n }\n}\n\nasync function checkAndUpdateUpstream(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n {\n replicaVersion,\n publications: subscribed,\n initialSyncContext,\n }: SubscriptionStateAndContext,\n replicaID: string | undefined,\n) {\n const upstreamReplica = await getReplicaAtVersion(\n lc,\n sql,\n shard,\n replicaVersion,\n replicaID,\n initialSyncContext,\n );\n if (!upstreamReplica) {\n throw new AutoResetSignal(\n `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n );\n }\n\n // Verify that the publications match what is being replicated.\n const requested = shard.publications.toSorted();\n const replicated = upstreamReplica.publications\n .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n .sort();\n if (!deepEqual(requested, replicated)) {\n lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n throw new AutoResetSignal(\n `Requested publications [${requested}] do not match configured ` +\n `publications: [${replicated}]`,\n );\n }\n\n // Sanity check: The subscription state on the replica should have the\n // same publications. This should be guaranteed by the equivalence of the\n // replicaVersion, but it doesn't hurt to verify.\n if (!deepEqual(upstreamReplica.publications, subscribed)) {\n throw new AutoResetSignal(\n `Upstream publications [${upstreamReplica.publications}] do not ` +\n `match subscribed publications [${subscribed}]`,\n );\n }\n\n // Verify that the publications exist.\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n `.values();\n if (exists.length !== subscribed.length) {\n throw new AutoResetSignal(\n `Upstream publications [${exists.flat()}] do not contain ` +\n `all subscribed publications [${subscribed}]`,\n );\n }\n\n const {slot} = upstreamReplica;\n const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n /*sql*/ `\n SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n WHERE slot_name = ${slot}`;\n if (result.length === 0) {\n throw new AutoResetSignal(`replication slot ${slot} is missing`);\n }\n const [{restartLSN, walStatus}] = result;\n if (restartLSN === null || walStatus === 'lost') {\n throw new AutoResetSignal(\n `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n );\n }\n const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgEA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,iCACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;AAChC,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,uBACA,iBACA,kBACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,MAAM,eAAe,SAAS,QAAQ;GAMtC,MAAM,WAAW,QAAQ;GACzB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;GAAQ,GAChB,eAAe,OACjB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAIA,MAAM,mBAAmB,YAAY,CAAC;EACtC,MAAM,aAAa,uBAEf,qBAAqB,aAIrB,WACG,iBAAiB,cAAc,OAAO,KAAK,IAAI,CAAC,IAChD,iBAAiB,cAAc;EA2BtC,OAAO;GACL;GACA,cAAA,IA3BuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;GAAQ,GACrB,SACA,qBACA,YAAY,UACZ,wBACA,kBACE,kBAAkB,WAClB,yBAAyB,KAAA,GACzB,eACA,SACF,CAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;GACA;GACA,WAAW,iBAAiB;EAC9B;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAmBA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,kBAAkB,WACL;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBAAgB,gBAAgB,WAAW,kBAAkB,WACjE;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,cAAc,mBAAmB,WAAW,OAAO;EACzD,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EAWA,OAAO;GAAC;GAAS;GAAW,iBAAA,MAVE,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,GACJ,OACF;EAC2C;CAC7C;CACA,OAAO;EAAC;EAAS;CAAS;AAC5B;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,kBAAkB,WAC/B,eAC0B;CAC1B,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,SAAS,gBAAe;CACzD,MAAM,gBAAgB,qBAAqB,OAAO;CAKlD,SAAS,UAAU,oBAAoB,YAAY,cACjD,YAAY,QAAQ,CACtB;CAkBA,MAAM,OAAO,mBAAmB,MAAM,iBAAiB,aAAa,IAAI;CACxE,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAIlD,MAAM,cAAc,mBAAmB,WAAW,WAAW;CAE7D,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EAWA,OAAO;GACL,SAAS;GACT;GACA;GACA,iBAAA,MAd4B,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,GACJ,OACF;GAME;EACF;CACF;CACA,OAAO;EACL,SAAS;EACT;EACA;EACA;CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;AA2BA,SAAgB,kBACd,kBACA,eACA,eACA,WACoB;CACpB,IAAI,eACF,OAAO;CAET,IAAI,kBAAkB,KAAA,GACpB,OAAO,mBAAmB,gBAAgB,mBAAmB;CAE/D,OAAO,WAAW;AACpB;AAEA,SAAS,mBACP,WACA,EAAC,cACmB;CACpB,OAAO,aAAa;EAAC,gBAAgB;EAAY,cAAc;CAAE;AACnE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AACjC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BAcjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;CAEJ,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,IAAI,KAAK,OAAO,KAAK;GACjE,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,aAAa,eAAe,QAAQ;IACpC;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IAAI,IAAI,OAAO,GACb,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;MACZ,SACM,aACN,CACF;MAQA,YAAY,MAAM,cAAc,GAAG;MACnC,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OACA;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;OAC1B,aAAa,YAAY;MAC3B;KACF;IACF;IAEA,IACE,QAAQ,UAAU,KAClB,QAAQ,UACR,QAAQ,OAAO,aAAa,QAAQ;SAGhC,IAAI,OAAO,GACb,GAAG,OACD,uBAAuB,QAAQ,GAAG,GAAG,QAAQ,KAAK,kCAClD,EAAC,QAAO,CACV;IAAA;IAIJ,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,wBAAwB,WAAW,KACzE,EAAC,QAAO,CACV;KACA,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,OACF;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,aAAa,eAAe,QAAQ;OACpC,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;QACjB,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,aAAa,eAAe,QAAQ;EACpC,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}
|
|
@@ -21,7 +21,13 @@ import { type BackupOptions, type InternalShardConfig, type Replica } from './sc
|
|
|
21
21
|
*/
|
|
22
22
|
export declare class PostgresChangeSource implements ChangeSource {
|
|
23
23
|
#private;
|
|
24
|
-
constructor(lc: LogContext, upstreamUri: string, shard: ShardID, replica: Replica, pgVersion: number, backupOptions: BackupOptions, context: ServerContext, lagReportIntervalMs: number, textCopy?: boolean | undefined, streamInboundTimeoutMs?: number | undefined,
|
|
24
|
+
constructor(lc: LogContext, upstreamUri: string, shard: ShardID, replica: Replica, pgVersion: number, backupOptions: BackupOptions, context: ServerContext, lagReportIntervalMs: number, textCopy?: boolean | undefined, streamInboundTimeoutMs?: number | undefined,
|
|
25
|
+
/**
|
|
26
|
+
* The watermark below which a stream cannot be started without skipping
|
|
27
|
+
* transactions, as Postgres would silently start from the slot's position
|
|
28
|
+
* instead (see `minStartWatermark()` in change-source-init.ts).
|
|
29
|
+
*/
|
|
30
|
+
minStartWatermark?: string | undefined, deps?: {
|
|
25
31
|
subscribe?: typeof subscribe;
|
|
26
32
|
streamBackfill?: typeof streamBackfill;
|
|
27
33
|
});
|