npm - @rocicorp/zero - Versions diffs - 1.10.0-canary.1 → 1.10.0-canary.3 - Mend

@rocicorp/zero 1.10.0-canary.1 → 1.10.0-canary.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (180) hide show

package/out/zero-cache/src/server/change-streamer.js CHANGED Viewed

@@ -1,4 +1,5 @@
 import { assert } from "../../../shared/src/asserts.js";
+import { promiseVoid } from "../../../shared/src/resolved-promises.js";
 import { must } from "../../../shared/src/must.js";
 import { childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardConfig } from "../types/shards.js";
@@ -36,7 +37,7 @@ async function runWorker(parent, env, ...argv) {
 		env,
 		argv
 	});
-	const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusTimeoutProportion, flowControlSlowSubscriberGracePeriodSeconds, sqliteChangeLogMode, sqliteChangeLogRetentionMs, sqliteChangeLogReadBatchRows, sqliteChangeLogPurgeBatchRows, sqliteChangeLogBarrierTimeoutMs }, autoReset, replicationLag, litestream, upstream, change, replica, initialSync, keepaliveTimeoutMs, sqliteCorruptionChecks } = config;
+	const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusTimeoutProportion, flowControlSlowSubscriberGracePeriodSeconds, sqliteChangeLogMode, sqliteChangeLogReadPercent, sqliteChangeLogColdReadPercent, sqliteChangeLogComparePercent, sqliteChangeLogRetentionMs, sqliteChangeLogReadBatchRows, sqliteChangeLogPurgeBatchRows, sqliteChangeLogBarrierTimeoutMs }, autoReset, replicationLag, litestream, upstream, change, replica, initialSync, keepaliveTimeoutMs, sqliteCorruptionChecks } = config;
 	startOtelAuto(createLogContext(config, "change-streamer", 0, false), "change-streamer", 0);
 	lc = createLogContext(config, "change-streamer");
 	registerSQLiteCorruptionDiagnosticTarget({
@@ -102,7 +103,17 @@ async function runWorker(parent, env, ...argv) {
 				retentionMs: sqliteChangeLogRetentionMs,
 				batchRows: sqliteChangeLogPurgeBatchRows
 			} : void 0,
-			sqliteChangeLogCompare: sqliteChangeLogComparing ? { replicaFile: replica.file } : void 0
+			sqliteChangeLogCompare: sqliteChangeLogComparing ? {
+				replicaFile: replica.file,
+				comparePercent: sqliteChangeLogComparePercent,
+				retentionMs: sqliteChangeLogRetentionMs,
+				readBatchRows: sqliteChangeLogReadBatchRows
+			} : void 0,
+			sqliteChangeLogServe: sqliteChangeLogMode === "serve" ? {
+				readPercent: sqliteChangeLogReadPercent,
+				coldReadPercent: sqliteChangeLogColdReadPercent,
+				retentionMs: sqliteChangeLogRetentionMs
+			} : void 0
 		}, setTimeout);
 		backupURL = destinationBackupURL;
 		waitForFirstBackupBeforeServing = waitForBackupBeforeServing;
@@ -133,26 +144,26 @@ async function runWorker(parent, env, ...argv) {
 	}
 	const backupMonitor = createBackupCleanupMonitor({
 		lc,
-		processes,
 		config,
 		replicaFile: replica.file,
-		changeStreamer,
-		env
+		changeStreamer
 	});
-	const changeStreamerWebServer = new ChangeStreamerHttpServer(lc, {
-		port,
-		keepaliveTimeoutMs,
-		startupDelayMs
-	}, parent, changeStreamer);
+	let readinessGate = promiseVoid;
 	if (waitForFirstBackupBeforeServing) {
 		const start = performance.now();
 		lc.info?.(`awaiting initial backup ...`);
-		backupMonitor.firstBackupReceived().then(() => {
+		readinessGate = backupMonitor.firstBackupReceived().then(() => {
 			const elapsed = performance.now() - start;
 			lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);
-			parent.send(["ready", { ready: true }]);
 		});
-	} else parent.send(["ready", { ready: true }]);
+	}
+	const changeStreamerWebServer = new ChangeStreamerHttpServer(lc, {
+		port,
+		keepaliveTimeoutMs,
+		startupDelayMs,
+		readinessGate
+	}, parent, changeStreamer);
+	readinessGate.then(() => parent.send(["ready", { ready: true }]));
 	try {
 		await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);
 	} catch (err) {

package/out/zero-cache/src/server/change-streamer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n changeLogFileName,\n deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodSeconds,\n sqliteChangeLogMode,\n sqliteChangeLogRetentionMs,\n sqliteChangeLogReadBatchRows,\n sqliteChangeLogPurgeBatchRows,\n sqliteChangeLogBarrierTimeoutMs,\n },\n autoReset,\n replicationLag,\n litestream,\n upstream,\n change,\n replica,\n initialSync,\n keepaliveTimeoutMs,\n sqliteCorruptionChecks,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n lc = createLogContext(config, 'change-streamer');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer replica',\n dbPath: replica.file,\n },\n sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Kick off DB connection warmup in the background.\n const changeDB = await connectPgClient(\n lc,\n change.db,\n 'change-streamer',\n {max: change.maxConns},\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date.\n await initChangeStreamerSchema(lc, changeDB, shard);\n\n // When restoring from litestream, acquire a lock to prevent change-log\n // purges. This ensures that (this) change-streamer will be able to resume\n // from the backup.\n let purgeLock =\n litestream.backupURL && litestream.executable\n ? await new PurgeLocker(lc, shard, changeDB).acquire()\n : null;\n const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n let changeStreamer: ChangeStreamerService \| undefined;\n let backupURL: string \| undefined;\n\n const context = getServerContext(config);\n const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n // The modes are cumulative, so `serve` implies `compare`.\n const sqliteChangeLogComparing =\n sqliteChangeLogMode === 'compare' \|\| sqliteChangeLogMode === 'serve';\n if (sqliteChangeLogEnabled) {\n const changeLogFile = changeLogFileName(replica.file);\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer change-log',\n dbPath: changeLogFile,\n },\n sqliteCorruptionChecks,\n );\n // The log's bytes are accounted for in the process that writes them. This\n // is local disk only — the log is excluded from the litestream backup — and\n // it is a whole-database total, because a table that left the replica left\n // through its wal: the replica's footprint is what shrank and this is where\n // it went.\n getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n description:\n `The SQLite change log's total on-disk footprint: its main db file ` +\n `plus its wal sidecars.`,\n unit: 'By',\n }).addCallback(async o =>\n o.observe(await sqliteFileBytes(lc, changeLogFile)),\n );\n }\n\n let waitForFirstBackupBeforeServing = false;\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {\n changeSource,\n subscriptionState,\n destinationBackupURL,\n replicaID,\n waitForBackupBeforeServing,\n } =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n },\n context,\n replicationLag.reportIntervalMs,\n restoreOptions,\n {backupV5: litestream.backupUsingV5},\n purgeLock,\n upstream.pgStreamInboundTimeoutMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n restoreOptions,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n destinationBackupURL\n ? {\n backupURL: destinationBackupURL,\n litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n }\n : null,\n purgeLock,\n autoReset ?? false,\n {\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n flowControlSlowSubscriberGracePeriodSeconds > 0\n ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n : undefined,\n statementTimeoutMs: change.statementTimeoutMs,\n changeLogBatchSize: change.logBatchSize,\n sqliteCatchup: {\n changeLogFile: changeLogFileName(replica.file),\n readBatchRows: sqliteChangeLogReadBatchRows,\n barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n },\n // The presence of these options is the writer's gate. This process\n // performs the restore and the initial sync, so the replica's\n // identity is in hand at the moment the log is opened.\n sqliteChangeLogWriter: sqliteChangeLogEnabled\n ? {\n replicaFile: replica.file,\n identity: {\n // RMv2 supplies the epoch; until then the generation and the\n // replica ID are the whole of the identity.\n epoch: null,\n generation: subscriptionState.replicaVersion,\n replicaID,\n },\n }\n : undefined,\n // The purge scheduler shares the writer's gate -- mode, not the\n // read path -- because `write` mode, with no read selector at all,\n // is the configuration it ships in.\n sqliteChangeLogPurge: sqliteChangeLogEnabled\n ? {\n retentionMs: sqliteChangeLogRetentionMs,\n batchRows: sqliteChangeLogPurgeBatchRows,\n }\n : undefined,\n // `compare` and above. The replica-derived initialization path runs\n // at full rate and is checked against Postgres, which stays\n // authoritative. Flipping authority to the replica comes later.\n sqliteChangeLogCompare: sqliteChangeLogComparing\n ? {replicaFile: replica.file}\n : undefined,\n },\n setTimeout,\n );\n backupURL = destinationBackupURL;\n waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // The change log carries the identity of the replica it was written\n // beside, and the retry performs a fresh initial sync with a new\n // replicaVersion. Reconciliation would catch that on its own (as\n // 'identity-mismatch') but only after the writer opened a file that is\n // known here to be garbage.\n deleteChangeLogDB(replica.file);\n // Release the purge lock before retrying. This is safe because the\n // purge lock exists to preserve change-log entries so the new\n // change-streamer can resume from the backup replica's watermark.\n // An AutoResetSignal means we cant resume from the backup replica\n // (e.g. its replication slot is gone), so the change-log entries the lock\n // was protecting are no longer needed. The retry performs a fresh\n // initial sync with a new replication slot, independent of the old\n // change-log. Releasing is also necessary to avoid a\n // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n // CREATE_REPLICATION_SLOT waits for all older transactions to\n // finish, including this lock's open transaction.\n await purgeLock?.release();\n purgeLock = null;\n continue; // execute again with a fresh initial-sync\n }\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n const processes = new ProcessManager(lc, parent);\n if (backupURL) {\n lc.info?.('setting up backup to', backupURL);\n litestream.backupURL = backupURL;\n const {promise: backupStarted, resolve} = resolver();\n\n // Start a backup replicator and corresponding litestream backup process.\n processes\n .addWorker(\n childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n 'supporting',\n 'backup-replicator',\n )\n // Wait for the replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the sqlite lock\n // when the replicator first prepares the db file.\n .once('message', () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, litestream, replica.file),\n 'supporting',\n 'litestream',\n );\n resolve();\n });\n await backupStarted;\n }\n\n const backupMonitor = createBackupCleanupMonitor({\n lc,\n processes,\n config,\n replicaFile: replica.file,\n changeStreamer,\n env,\n });\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n {port, keepaliveTimeoutMs, startupDelayMs},\n parent,\n changeStreamer,\n );\n\n if (waitForFirstBackupBeforeServing) {\n const start = performance.now();\n lc.info?.(`awaiting initial backup ...`);\n\n void backupMonitor.firstBackupReceived().then(() => {\n const elapsed = performance.now() - start;\n lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n parent.send(['ready', {ready: true}]);\n });\n } else {\n parent.send(['ready', {ready: true}]);\n }\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer after a delay to ensure that routing\n // routing elements have registered the server before the\n // change-streamer takes over the replication slot.\n // TODO: Remove this delay and start the changeStreamer normally here once\n // transitioned to RMv2, since changeStreamer startup will no longer\n // disrupt an existing task.\n try {\n await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n } catch (err) {\n processes.logErrorAndExit(err, 'change-streamer');\n } finally {\n await processes.shutdown();\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () =>\n runWorker(\n must(parentWorker),\n process.env,\n ...process.argv.slice(2),\n ).catch(async e => {\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n throw e;\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,qBACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAGxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,mBACA,EAAC,KAAK,OAAO,SAAQ,GACrB,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,WAAW,aAAa,WAAW,aAC/B,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;EACpC,GACA,SACA,eAAe,kBACf,gBACA,EAAC,UAAU,WAAW,cAAa,GACnC,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAIJ,wBAAwB,2BACpB,EAAC,aAAa,QAAQ,KAAI,IAC1B,KAAA;EACN,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,UACG,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA;EACA,aAAa,QAAQ;EACrB;EACA;CACF,CAAC;CAED,MAAM,0BAA0B,IAAI,yBAClC,IACA;EAAC;EAAM;EAAoB;CAAc,GACzC,QACA,cACF;CAEA,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAmB,oBAAoB,EAAE,WAAW;GAClD,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;GAClE,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACtC,CAAC;CACH,OACE,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAUtC,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
1	+ {"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n changeLogFileName,\n deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodSeconds,\n sqliteChangeLogMode,\n sqliteChangeLogReadPercent,\n sqliteChangeLogColdReadPercent,\n sqliteChangeLogComparePercent,\n sqliteChangeLogRetentionMs,\n sqliteChangeLogReadBatchRows,\n sqliteChangeLogPurgeBatchRows,\n sqliteChangeLogBarrierTimeoutMs,\n },\n autoReset,\n replicationLag,\n litestream,\n upstream,\n change,\n replica,\n initialSync,\n keepaliveTimeoutMs,\n sqliteCorruptionChecks,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n lc = createLogContext(config, 'change-streamer');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer replica',\n dbPath: replica.file,\n },\n sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Kick off DB connection warmup in the background.\n const changeDB = await connectPgClient(\n lc,\n change.db,\n 'change-streamer',\n {max: change.maxConns},\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date.\n await initChangeStreamerSchema(lc, changeDB, shard);\n\n // When restoring from litestream, acquire a lock to prevent change-log\n // purges. This ensures that (this) change-streamer will be able to resume\n // from the backup.\n let purgeLock =\n litestream.backupURL && litestream.executable\n ? await new PurgeLocker(lc, shard, changeDB).acquire()\n : null;\n const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n let changeStreamer: ChangeStreamerService \| undefined;\n let backupURL: string \| undefined;\n\n const context = getServerContext(config);\n const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n // The modes are cumulative, so `serve` implies `compare`.\n const sqliteChangeLogComparing =\n sqliteChangeLogMode === 'compare' \|\| sqliteChangeLogMode === 'serve';\n if (sqliteChangeLogEnabled) {\n const changeLogFile = changeLogFileName(replica.file);\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer change-log',\n dbPath: changeLogFile,\n },\n sqliteCorruptionChecks,\n );\n // The log's bytes are accounted for in the process that writes them. This\n // is local disk only — the log is excluded from the litestream backup — and\n // it is a whole-database total, because a table that left the replica left\n // through its wal: the replica's footprint is what shrank and this is where\n // it went.\n getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n description:\n `The SQLite change log's total on-disk footprint: its main db file ` +\n `plus its wal sidecars.`,\n unit: 'By',\n }).addCallback(async o =>\n o.observe(await sqliteFileBytes(lc, changeLogFile)),\n );\n }\n\n let waitForFirstBackupBeforeServing = false;\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {\n changeSource,\n subscriptionState,\n destinationBackupURL,\n replicaID,\n waitForBackupBeforeServing,\n } =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n },\n context,\n replicationLag.reportIntervalMs,\n restoreOptions,\n {backupV5: litestream.backupUsingV5},\n purgeLock,\n upstream.pgStreamInboundTimeoutMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n restoreOptions,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n destinationBackupURL\n ? {\n backupURL: destinationBackupURL,\n litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n }\n : null,\n purgeLock,\n autoReset ?? false,\n {\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n flowControlSlowSubscriberGracePeriodSeconds > 0\n ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n : undefined,\n statementTimeoutMs: change.statementTimeoutMs,\n changeLogBatchSize: change.logBatchSize,\n sqliteCatchup: {\n changeLogFile: changeLogFileName(replica.file),\n readBatchRows: sqliteChangeLogReadBatchRows,\n barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n },\n // The presence of these options is the writer's gate. This process\n // performs the restore and the initial sync, so the replica's\n // identity is in hand at the moment the log is opened.\n sqliteChangeLogWriter: sqliteChangeLogEnabled\n ? {\n replicaFile: replica.file,\n identity: {\n // RMv2 supplies the epoch; until then the generation and the\n // replica ID are the whole of the identity.\n epoch: null,\n generation: subscriptionState.replicaVersion,\n replicaID,\n },\n }\n : undefined,\n // The purge scheduler shares the writer's gate -- mode, not the\n // read path -- because `write` mode, with no read selector at all,\n // is the configuration it ships in.\n sqliteChangeLogPurge: sqliteChangeLogEnabled\n ? {\n retentionMs: sqliteChangeLogRetentionMs,\n batchRows: sqliteChangeLogPurgeBatchRows,\n }\n : undefined,\n // Compare mode runs both advisory checks. Postgres remains authoritative.\n sqliteChangeLogCompare: sqliteChangeLogComparing\n ? {\n replicaFile: replica.file,\n comparePercent: sqliteChangeLogComparePercent,\n retentionMs: sqliteChangeLogRetentionMs,\n readBatchRows: sqliteChangeLogReadBatchRows,\n }\n : undefined,\n // Slice 11 lands dark by default: serve mode constructs the stable\n // router, while readPercent=0 keeps every catchup on PG and emits\n // eligibility metrics before any canary traffic is enabled.\n sqliteChangeLogServe:\n sqliteChangeLogMode === 'serve'\n ? {\n readPercent: sqliteChangeLogReadPercent,\n coldReadPercent: sqliteChangeLogColdReadPercent,\n retentionMs: sqliteChangeLogRetentionMs,\n }\n : undefined,\n },\n setTimeout,\n );\n backupURL = destinationBackupURL;\n waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // The change log carries the identity of the replica it was written\n // beside, and the retry performs a fresh initial sync with a new\n // replicaVersion. Reconciliation would catch that on its own (as\n // 'identity-mismatch') but only after the writer opened a file that is\n // known here to be garbage.\n deleteChangeLogDB(replica.file);\n // Release the purge lock before retrying. This is safe because the\n // purge lock exists to preserve change-log entries so the new\n // change-streamer can resume from the backup replica's watermark.\n // An AutoResetSignal means we cant resume from the backup replica\n // (e.g. its replication slot is gone), so the change-log entries the lock\n // was protecting are no longer needed. The retry performs a fresh\n // initial sync with a new replication slot, independent of the old\n // change-log. Releasing is also necessary to avoid a\n // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n // CREATE_REPLICATION_SLOT waits for all older transactions to\n // finish, including this lock's open transaction.\n await purgeLock?.release();\n purgeLock = null;\n continue; // execute again with a fresh initial-sync\n }\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n const processes = new ProcessManager(lc, parent);\n if (backupURL) {\n lc.info?.('setting up backup to', backupURL);\n litestream.backupURL = backupURL;\n const {promise: backupStarted, resolve} = resolver();\n\n // Start a backup replicator and corresponding litestream backup process.\n processes\n .addWorker(\n childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n 'supporting',\n 'backup-replicator',\n )\n // Wait for the replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the sqlite lock\n // when the replicator first prepares the db file.\n .once('message', () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, litestream, replica.file),\n 'supporting',\n 'litestream',\n );\n resolve();\n });\n await backupStarted;\n }\n\n const backupMonitor = createBackupCleanupMonitor({\n lc,\n config,\n replicaFile: replica.file,\n changeStreamer,\n });\n\n let readinessGate = promiseVoid;\n if (waitForFirstBackupBeforeServing) {\n const start = performance.now();\n lc.info?.(`awaiting initial backup ...`);\n\n readinessGate = backupMonitor.firstBackupReceived().then(() => {\n const elapsed = performance.now() - start;\n lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n });\n }\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n {port, keepaliveTimeoutMs, startupDelayMs, readinessGate},\n parent,\n changeStreamer,\n );\n\n void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer after a delay to ensure that routing\n // routing elements have registered the server before the\n // change-streamer takes over the replication slot.\n // TODO: Remove this delay and start the changeStreamer normally here once\n // transitioned to RMv2, since changeStreamer startup will no longer\n // disrupt an existing task.\n try {\n await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n } catch (err) {\n processes.logErrorAndExit(err, 'change-streamer');\n } finally {\n await processes.shutdown();\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () =>\n runWorker(\n must(parentWorker),\n process.env,\n ...process.argv.slice(2),\n ).catch(async e => {\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n throw e;\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAGxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,mBACA,EAAC,KAAK,OAAO,SAAQ,GACrB,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,WAAW,aAAa,WAAW,aAC/B,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;EACpC,GACA,SACA,eAAe,kBACf,gBACA,EAAC,UAAU,WAAW,cAAa,GACnC,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBAAwB,2BACpB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIJ,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,UACG,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,MAAM,0BAA0B,IAAI,yBAClC,IACA;EAAC;EAAM;EAAoB;EAAgB;CAAa,GACxD,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}

package/out/zero-cache/src/server/runner/run-worker.js CHANGED Viewed

@@ -36,7 +36,7 @@ async function runWorker(parent, env) {
 	const { port, keepaliveTimeoutMs, lazyStartup } = config;
 	const serverVersion = getServerVersion(config);
 	lc.info?.(`starting server${!serverVersion ? "" : `@${serverVersion}`} `, {
-		protocolVersion: 51,
+		protocolVersion: 52,
 		taskID: config.taskID,
 		app: config.app,
 		shard: config.shard,

package/out/zero-cache/src/server/worker-urls.d.ts CHANGED Viewed

@@ -1,5 +1,4 @@
 export declare const CHANGE_STREAMER_URL: URL;
-export declare const BACKUP_WATERMARK_POLLER_URL: URL;
 export declare const MAIN_URL: URL;
 export declare const MUTATOR_URL: URL;
 export declare const REAPER_URL: URL;

package/out/zero-cache/src/server/worker-urls.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"worker-urls.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,mBAAmB,KAAkC,CAAC;AACnE,eAAO,MAAM,~~2BAA2B,KAEvC,CAAC;AACF,eAAO,MAAM,~~QAAQ,KAAuB,CAAC;AAC7C,eAAO,MAAM,WAAW,KAA0B,CAAC;AACnD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,cAAc,KAA6B,CAAC;AACzD,eAAO,MAAM,iBAAiB,KAAgC,CAAC;AAC/D,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,gBAAgB,KAA+B,CAAC"}
1	+ {"version":3,"file":"worker-urls.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,mBAAmB,KAAkC,CAAC;AACnE,eAAO,MAAM,QAAQ,KAAuB,CAAC;AAC7C,eAAO,MAAM,WAAW,KAA0B,CAAC;AACnD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,cAAc,KAA6B,CAAC;AACzD,eAAO,MAAM,iBAAiB,KAAgC,CAAC;AAC/D,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,gBAAgB,KAA+B,CAAC"}

package/out/zero-cache/src/server/worker-urls.js CHANGED Viewed

@@ -6,7 +6,6 @@ function resolve(path) {
 	return new URL(path, url);
 }
 var CHANGE_STREAMER_URL = resolve("./change-streamer.ts");
-var BACKUP_WATERMARK_POLLER_URL = resolve("./backup-watermark-poller.ts");
 var MAIN_URL = resolve("./main.ts");
 resolve("./mutator.ts");
 var REAPER_URL = resolve("./reaper.ts");
@@ -15,6 +14,6 @@ var SHADOW_SYNCER_URL = resolve("./shadow-syncer.ts");
 var SYNCER_URL = resolve("./syncer.ts");
 var WRITE_WORKER_URL = resolve("./write-worker.ts");
 //#endregion
-export { BACKUP_WATERMARK_POLLER_URL, CHANGE_STREAMER_URL, MAIN_URL, REAPER_URL, REPLICATOR_URL, SHADOW_SYNCER_URL, SYNCER_URL, WRITE_WORKER_URL };
+export { CHANGE_STREAMER_URL, MAIN_URL, REAPER_URL, REPLICATOR_URL, SHADOW_SYNCER_URL, SYNCER_URL, WRITE_WORKER_URL };
 //# sourceMappingURL=worker-urls.js.map

package/out/zero-cache/src/server/worker-urls.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n const {url} = import.meta;\n if (url.endsWith('.js')) {\n // When compiled, change .ts to .js\n path = path.replace(tsRe, '.js');\n }\n return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const ~~BACKUP_WATERMARK_POLLER_URL = resolve(\n './backup-watermark-poller.ts',\n);\nexport const~~ MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO,OAAO;CACrB,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,~~8BAA8B,QACzC,8BACF;AACA,IAAa,~~WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}
1	+ {"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n const {url} = import.meta;\n if (url.endsWith('.js')) {\n // When compiled, change .ts to .js\n path = path.replace(tsRe, '.js');\n }\n return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO,OAAO;CACrB,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js CHANGED Viewed

@@ -148,7 +148,7 @@ async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
 			mode: READONLY,
 			init
 		}).run(db);
-		await imported;
+		await imported.dequeue();
 		const watermark = toStateVersionString(lsn);
 		lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);
 		return {

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n DownloadStatus,\n JSONValue,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n makeBinarySelectExprs,\n makeDownloadStatements,\n type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n /*\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number \| undefined;\n\n /\n Use text-format COPY instead of binary COPY.\n * Binary is faster and handles all types (unknown types are cast to\n * `::text` in the SELECT). This flag exists as an escape hatch to\n * revert to the old code path if needed.\n /\n textCopy?: boolean \| undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b\|\\sLIMIT\\s+\\d/i;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n opts;\n const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n });\n let tx: TransactionPool \| undefined;\n let watermark: string;\n try {\n ({tx, watermark} = await createSnapshotTransaction(\n lc,\n upstreamURI,\n db,\n slot,\n ));\n const {tableSpec, backfill} = await validateSchema(\n tx,\n publications,\n bf,\n watermark,\n );\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n const stmts = makeDownloadStatements(tableSpec, cols);\n\n if (textCopy) {\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${stmts.select}) TO STDOUT`,\n new TsvParser(),\n cols.map(col => {\n const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n return (text: string) => parser(text) as JSONValue;\n }),\n flushThresholdBytes,\n );\n } else {\n const binaryStmts = makeDownloadStatements(\n tableSpec,\n cols,\n undefined,\n undefined,\n makeBinarySelectExprs(tableSpec, cols),\n );\n\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n new BinaryCopyParser(),\n cols.map(col => {\n const spec = tableSpec.columns[col];\n const decoder = hasBinaryDecoder(spec)\n ? makeBinaryDecoder(spec)\n : textCastDecoder;\n return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n }),\n flushThresholdBytes,\n );\n }\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx?.setDone();\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream<T>(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n {\n getTotalRows,\n getTotalBytes,\n }: Pick<DownloadStatements, 'getTotalRows' \| 'getTotalBytes'>,\n copyCommand: string,\n parser: {parse(chunk: Buffer): Iterable<T \| null>},\n decoders: ((field: T) => JSONValue)[],\n flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n // sync and must never appear in a backfill COPY.\n assert(\n !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n `backfill COPY must not sample or limit: ${copyCommand}`,\n );\n const start = performance.now();\n const [rows, bytes] = await tx.processReadTask(sql =>\n Promise.all([\n sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n ]),\n );\n const status: DownloadStatus = {\n rows: 0,\n totalRows: Number(rows[0].totalRows),\n totalBytes: Number(bytes[0].totalBytes),\n };\n\n let elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n {\n status,\n },\n );\n const copyStream = await tx.processReadTask(sql =>\n sql.unsafe(copyCommand).readable(),\n );\n\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: decoders.length});\n let col = 0;\n\n for await (const data of copyStream) {\n const chunk = data as Buffer;\n for (const field of parser.parse(chunk)) {\n row[col] = field === null ? null : decoders[col](field);\n\n if (++col === decoders.length) {\n rowValues.push(row);\n status.rows++;\n row = Array.from({length: decoders.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n }\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n }\n\n yield {tag: 'backfill-completed', ...backfill, status};\n elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n */\nasync function createSnapshotTransaction(\n lc: LogContext,\n upstreamURI: string,\n db: PostgresDB,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(\n lc,\n upstreamURI,\n 'backfill-replication-session',\n {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n },\n );\n const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, {\n slotName,\n temporary: true, // deletes the slot when the replicationSession ends\n });\n\n const {init, imported} = importSnapshot(snapshot);\n const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n await imported;\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return {tx, watermark};\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n return {tableSpec: spec, backfill};\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAgEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GAC8B;CACrD,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB,GAClE,iBAAiB,KACpB,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EACpD,MAAM,QAAQ,uBAAuB,WAAW,IAAI;EAEpD,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,GACD,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,IAAI,CACvC;GAEA,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,GACD,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACqD;CAGrD,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CACA,MAAM,aAAa,MAAM,GAAG,iBAAgB,QAC1C,IAAI,OAAO,WAAW,EAAE,SAAS,CACnC;CAEA,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAGA,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,WAAW,MAAM,QAAQ,YAAY;EACnC,MAAM,QAAQ;EACd,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;GAEtD,IAAI,EAAE,QAAQ,SAAS,QAAQ;IAC7B,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,qBAAqB;GACxC,MAAM;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;GAAM;GACtD;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;CACF;CAGA,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GAAC,KAAK;GAAY,GAAG;GAAU;GAAW;EAAM;EACtD;EACA,WAAW;CACb;CAEA,MAAM;EAAC,KAAK;EAAsB,GAAG;EAAU;CAAM;CACrD,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;EACb,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM;EAEN,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,CAC9B;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAYA,OAAO;GAAC,WAAW;GAAM,UAAA;IAVvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAEuB;EAAQ;CACnC,CAAC;AACH"}
1	+ {"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n DownloadStatus,\n JSONValue,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n makeBinarySelectExprs,\n makeDownloadStatements,\n type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n /*\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number \| undefined;\n\n /\n Use text-format COPY instead of binary COPY.\n * Binary is faster and handles all types (unknown types are cast to\n * `::text` in the SELECT). This flag exists as an escape hatch to\n * revert to the old code path if needed.\n /\n textCopy?: boolean \| undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b\|\\sLIMIT\\s+\\d/i;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n opts;\n const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n });\n let tx: TransactionPool \| undefined;\n let watermark: string;\n try {\n ({tx, watermark} = await createSnapshotTransaction(\n lc,\n upstreamURI,\n db,\n slot,\n ));\n const {tableSpec, backfill} = await validateSchema(\n tx,\n publications,\n bf,\n watermark,\n );\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n const stmts = makeDownloadStatements(tableSpec, cols);\n\n if (textCopy) {\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${stmts.select}) TO STDOUT`,\n new TsvParser(),\n cols.map(col => {\n const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n return (text: string) => parser(text) as JSONValue;\n }),\n flushThresholdBytes,\n );\n } else {\n const binaryStmts = makeDownloadStatements(\n tableSpec,\n cols,\n undefined,\n undefined,\n makeBinarySelectExprs(tableSpec, cols),\n );\n\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n new BinaryCopyParser(),\n cols.map(col => {\n const spec = tableSpec.columns[col];\n const decoder = hasBinaryDecoder(spec)\n ? makeBinaryDecoder(spec)\n : textCastDecoder;\n return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n }),\n flushThresholdBytes,\n );\n }\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx?.setDone();\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream<T>(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n {\n getTotalRows,\n getTotalBytes,\n }: Pick<DownloadStatements, 'getTotalRows' \| 'getTotalBytes'>,\n copyCommand: string,\n parser: {parse(chunk: Buffer): Iterable<T \| null>},\n decoders: ((field: T) => JSONValue)[],\n flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n // sync and must never appear in a backfill COPY.\n assert(\n !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n `backfill COPY must not sample or limit: ${copyCommand}`,\n );\n const start = performance.now();\n const [rows, bytes] = await tx.processReadTask(sql =>\n Promise.all([\n sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n ]),\n );\n const status: DownloadStatus = {\n rows: 0,\n totalRows: Number(rows[0].totalRows),\n totalBytes: Number(bytes[0].totalBytes),\n };\n\n let elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n {\n status,\n },\n );\n const copyStream = await tx.processReadTask(sql =>\n sql.unsafe(copyCommand).readable(),\n );\n\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: decoders.length});\n let col = 0;\n\n for await (const data of copyStream) {\n const chunk = data as Buffer;\n for (const field of parser.parse(chunk)) {\n row[col] = field === null ? null : decoders[col](field);\n\n if (++col === decoders.length) {\n rowValues.push(row);\n status.rows++;\n row = Array.from({length: decoders.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n }\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n }\n\n yield {tag: 'backfill-completed', ...backfill, status};\n elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n */\nasync function createSnapshotTransaction(\n lc: LogContext,\n upstreamURI: string,\n db: PostgresDB,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(\n lc,\n upstreamURI,\n 'backfill-replication-session',\n {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n },\n );\n const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, {\n slotName,\n temporary: true, // deletes the slot when the replicationSession ends\n });\n\n const {init, imported} = importSnapshot(snapshot);\n const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n await imported.dequeue();\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return {tx, watermark};\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n return {tableSpec: spec, backfill};\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAgEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GAC8B;CACrD,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB,GAClE,iBAAiB,KACpB,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EACpD,MAAM,QAAQ,uBAAuB,WAAW,IAAI;EAEpD,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,GACD,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,IAAI,CACvC;GAEA,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,GACD,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACqD;CAGrD,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CACA,MAAM,aAAa,MAAM,GAAG,iBAAgB,QAC1C,IAAI,OAAO,WAAW,EAAE,SAAS,CACnC;CAEA,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAGA,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,WAAW,MAAM,QAAQ,YAAY;EACnC,MAAM,QAAQ;EACd,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;GAEtD,IAAI,EAAE,QAAQ,SAAS,QAAQ;IAC7B,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,qBAAqB;GACxC,MAAM;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;GAAM;GACtD;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;CACF;CAGA,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GAAC,KAAK;GAAY,GAAG;GAAU;GAAW;EAAM;EACtD;EACA,WAAW;CACb;CAEA,MAAM;EAAC,KAAK;EAAsB,GAAG;EAAU;CAAM;CACrD,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;EACb,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM,SAAS,QAAQ;EAEvB,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,CAC9B;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAYA,OAAO;GAAC,WAAW;GAAM,UAAA;IAVvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAEuB;EAAQ;CACnC,CAAC;AACH"}

package/out/zero-cache/src/services/change-source/pg/change-source.js CHANGED Viewed

@@ -25,11 +25,11 @@ import { initReplica } from "../common/replica-schema.js";
 import "../../../types/pg-versions.js";
 import { BackfillManager } from "../common/backfill-manager.js";
 import { ChangeStreamMultiplexer } from "../common/change-stream-multiplexer.js";
+import { subscribe } from "./logical-replication/stream.js";
 import { dropOldReplicasAndSlots } from "./replication-slots.js";
 import { ensureShardSchema } from "./schema/init.js";
 import { initialSync } from "./initial-sync.js";
 import { streamBackfill } from "./backfill-stream.js";
-import { subscribe } from "./logical-replication/stream.js";
 import { registerReplicationSlotHealthMetrics } from "./replication-slot-health.js";
 import { nanoid } from "nanoid";
 import postgres from "postgres";

package/out/zero-cache/src/services/change-source/pg/decommission.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"decommission.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/decommission.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;~~AAEjD~~,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sBAAsB,CAAC;~~AAGrD~~,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GAAG,MAAM,~~iBAiCzB~~"}
1	+ {"version":3,"file":"decommission.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/decommission.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sBAAsB,CAAC;AAKrD,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GAAG,MAAM,iBAiDzB"}

package/out/zero-cache/src/services/change-source/pg/decommission.js CHANGED Viewed

@@ -1,28 +1,37 @@
+import { sleep } from "../../../../../shared/src/sleep.js";
 import { runTx } from "../../../db/run-transaction.js";
 import { dropShard } from "./schema/shard.js";
 //#region ../zero-cache/src/services/change-source/pg/decommission.ts
+var MAX_ATTEMPTS = 3;
 async function decommissionShard(lc, db, appID, shardID) {
 	const shard = `${appID}_${shardID}`;
 	lc.info?.(`Decommissioning zero shard ${shard}`);
-	await runTx(db, async (sql) => {
-		const slots = await sql`
+	for (let i = 1;; i++) try {
+		await runTx(db, async (sql) => {
+			const slots = await sql`
     SELECT pg_terminate_backend(active_pid), active_pid as pid
       FROM pg_replication_slots
       WHERE slot_name = ${shard}
          OR slot_name LIKE ${shard + "_%"}`;
-		if (slots.length > 0) {
-			if (slots[0].pid !== null) lc.info?.(`signaled subscriber ${slots[0].pid} to shut down`);
-			const dropped = await sql`
+			if (slots.length > 0) {
+				if (slots[0].pid !== null) lc.info?.(`signaled subscriber ${slots[0].pid} to shut down`);
+				const dropped = await sql`
         SELECT pg_drop_replication_slot(slot_name), slot_name as "slotName"
           FROM pg_replication_slots
           WHERE slot_name = ${shard}
              OR slot_name LIKE ${`${appID}_${shardID}_%`.replaceAll("_", "\\_")}`;
-			lc.debug?.(`Dropped replication slot(s) ${dropped.map(({ slotName }) => slotName)}`);
-			await sql.unsafe(dropShard(appID, shardID));
-			lc.debug?.(`Dropped upstream shard schema ${shard} and event triggers`);
-		}
-	});
-	lc.info?.(`Finished decommissioning zero shard ${shard}`);
+				lc.debug?.(`Dropped replication slot(s) ${dropped.map(({ slotName }) => slotName)}`);
+				await sql.unsafe(dropShard(appID, shardID));
+				lc.debug?.(`Dropped upstream shard schema ${shard} and event triggers`);
+			}
+		});
+		lc.info?.(`Finished decommissioning zero shard ${shard}`);
+		return;
+	} catch (e) {
+		lc.warn?.(`error decommissioning shard`, e);
+		if (i === MAX_ATTEMPTS) throw e;
+		else await sleep(3e3);
+	}
 }
 //#endregion
 export { decommissionShard };

package/out/zero-cache/src/services/change-source/pg/decommission.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"decommission.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/decommission.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\nimport {dropShard} from './schema/shard.ts';\n\nexport async function decommissionShard(\n lc: LogContext,\n db: PostgresDB,\n appID: string,\n shardID: string \| number,\n) {\n const shard = `${appID}_${shardID}`;\n\n lc.info?.(`Decommissioning zero shard ${shard}`);\n await runTx(db, async sql => {\n // Kill the active_pid's on existing slots before altering publications,\n // as deleting a publication associated with an existing subscriber causes\n // weirdness; the active_pid becomes null and thus unable to be terminated.\n const slots = await sql<{pid: string \| null}[]>`\n SELECT pg_terminate_backend(active_pid), active_pid as pid\n FROM pg_replication_slots \n WHERE slot_name = ${shard} \n OR slot_name LIKE ${shard + '_%'}`;\n if (slots.length > 0) {\n if (slots[0].pid !== null) {\n lc.info?.(`signaled subscriber ${slots[0].pid} to shut down`);\n }\n // Escape underscores for the LIKE expression.\n const slotExpression = `${appID}_${shardID}_%`.replaceAll('_', '\\\\_');\n const dropped = await sql<{slotName: string}[]>`\n SELECT pg_drop_replication_slot(slot_name), slot_name as \"slotName\"\n FROM pg_replication_slots \n WHERE slot_name = ${shard} \n OR slot_name LIKE ${slotExpression}`;\n lc.debug?.(\n `Dropped replication slot(s) ${dropped.map(({slotName}) => slotName)}`,\n );\n await sql.unsafe(dropShard(appID, shardID));\n lc.debug?.(`Dropped upstream shard schema ${shard} and event triggers`);\n }\n });\n lc.info?.(`Finished decommissioning zero shard ${shard}`);\n}\n"],"mappings":"~~;;;AAKA~~,eAAsB,kBACpB,IACA,IACA,OACA,SACA;CACA,MAAM,QAAQ,GAAG,MAAM,GAAG;CAE1B,GAAG,OAAO,8BAA8B,OAAO;CAC/C,MAAM,MAAM,IAAI,OAAM,QAAO;~~EAI3B~~,MAAM,QAAQ,MAAM,GAA2B;;;~~0BAGzB~~,MAAM;6BACH,QAAQ;~~EACjC~~,IAAI,MAAM,SAAS,GAAG;~~GACpB~~,IAAI,MAAM,GAAG,QAAQ,MACnB,GAAG,OAAO,uBAAuB,MAAM,GAAG,IAAI,cAAc;~~GAI9D~~,MAAM,UAAU,MAAM,GAAyB;;;~~8BAGvB~~,MAAM;~~iCAJP~~,GAAG,MAAM,GAAG,QAAQ,IAAI,WAAW,KAAK,~~KAKpC~~;~~GAC3B~~,GAAG,QACD,+BAA+B,QAAQ,KAAK,EAAC,eAAc,QAAQ,GACrE;~~GACA~~,MAAM,IAAI,OAAO,UAAU,OAAO,OAAO,CAAC;~~GAC1C~~,GAAG,~~QAAQ~~,iCAAiC,MAAM,~~oBAAoB~~;~~EACxE~~;~~CACF~~,CAAC;~~CACD~~,GAAG,OAAO,uCAAuC,OAAO;~~AAC1D~~"}
1	+ {"version":3,"file":"decommission.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/decommission.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\nimport {dropShard} from './schema/shard.ts';\n\nconst MAX_ATTEMPTS = 3;\n\nexport async function decommissionShard(\n lc: LogContext,\n db: PostgresDB,\n appID: string,\n shardID: string \| number,\n) {\n const shard = `${appID}_${shardID}`;\n\n lc.info?.(`Decommissioning zero shard ${shard}`);\n for (let i = 1; ; i++) {\n try {\n await runTx(db, async sql => {\n // Kill the active_pid's on existing slots before altering publications,\n // as deleting a publication associated with an existing subscriber causes\n // weirdness; the active_pid becomes null and thus unable to be terminated.\n const slots = await sql<{pid: string \| null}[]>`\n SELECT pg_terminate_backend(active_pid), active_pid as pid\n FROM pg_replication_slots \n WHERE slot_name = ${shard} \n OR slot_name LIKE ${shard + '_%'}`;\n if (slots.length > 0) {\n if (slots[0].pid !== null) {\n lc.info?.(`signaled subscriber ${slots[0].pid} to shut down`);\n }\n // Escape underscores for the LIKE expression.\n const slotExpression = `${appID}_${shardID}_%`.replaceAll('_', '\\\\_');\n const dropped = await sql<{slotName: string}[]>`\n SELECT pg_drop_replication_slot(slot_name), slot_name as \"slotName\"\n FROM pg_replication_slots \n WHERE slot_name = ${shard} \n OR slot_name LIKE ${slotExpression}`;\n lc.debug?.(\n `Dropped replication slot(s) ${dropped.map(({slotName}) => slotName)}`,\n );\n await sql.unsafe(dropShard(appID, shardID));\n lc.debug?.(\n `Dropped upstream shard schema ${shard} and event triggers`,\n );\n }\n });\n lc.info?.(`Finished decommissioning zero shard ${shard}`);\n return;\n } catch (e) {\n lc.warn?.(`error decommissioning shard`, e);\n if (i === MAX_ATTEMPTS) {\n throw e;\n } else {\n // pg_terminate_backend is not transactional, which results in the\n // slot to sometimes fail to be dropped. Wait and retry.\n await sleep(3000);\n }\n }\n }\n}\n"],"mappings":";;;;AAMA,IAAM,eAAe;AAErB,eAAsB,kBACpB,IACA,IACA,OACA,SACA;CACA,MAAM,QAAQ,GAAG,MAAM,GAAG;CAE1B,GAAG,OAAO,8BAA8B,OAAO;CAC/C,KAAK,IAAI,IAAI,IAAK,KAChB,IAAI;EACF,MAAM,MAAM,IAAI,OAAM,QAAO;GAI3B,MAAM,QAAQ,MAAM,GAA2B;;;0BAG7B,MAAM;6BACH,QAAQ;GAC7B,IAAI,MAAM,SAAS,GAAG;IACpB,IAAI,MAAM,GAAG,QAAQ,MACnB,GAAG,OAAO,uBAAuB,MAAM,GAAG,IAAI,cAAc;IAI9D,MAAM,UAAU,MAAM,GAAyB;;;8BAG3B,MAAM;iCAJH,GAAG,MAAM,GAAG,QAAQ,IAAI,WAAW,KAAK,KAKxC;IACvB,GAAG,QACD,+BAA+B,QAAQ,KAAK,EAAC,eAAc,QAAQ,GACrE;IACA,MAAM,IAAI,OAAO,UAAU,OAAO,OAAO,CAAC;IAC1C,GAAG,QACD,iCAAiC,MAAM,oBACzC;GACF;EACF,CAAC;EACD,GAAG,OAAO,uCAAuC,OAAO;EACxD;CACF,SAAS,GAAG;EACV,GAAG,OAAO,+BAA+B,CAAC;EAC1C,IAAI,MAAM,cACR,MAAM;OAIN,MAAM,MAAM,GAAI;CAEpB;AAEJ"}

package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;~~AAGjD~~,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGzE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAoBzD,OAAO,KAAK,EAAC,SAAS,EAAE,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAaxE,OAAO,~~EAGL~~,KAAK,UAAU,EAGhB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAU1D,OAAO,EAML,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC/B,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9C;;;;;;OAMG;IACH,MAAM,CAAC,EACH;QACE,6DAA6D;QAC7D,UAAU,EAAE,MAAM,CAAC;QACnB;;;WAGG;QACH,eAAe,EAAE,MAAM,CAAC;KACzB,GACD,SAAS,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,EAAC,QAAQ,EAAC,GAAE,cAAiC,GAC5C,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAuRnC;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,aAAa,EACtB,WAAW,CAAC,EAAE,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,GACjD,OAAO,CAAC,IAAI,CAAC,CA+Bf;~~AAwMD~~;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE;IAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAAC,OAAO,EAAE,SAAS,EAAE,CAAA;CAAC,EAC/D,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GACvC,IAAI,CAgFN;AAMD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAMpC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAwBF;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,GACb,MAAM,EAAE,CAKV;AAED,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,EACd,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,EAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,EACpC,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,GACjC,kBAAkB,CA8BpB;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,cAAc,CAAC;CACxB,CAAC;AAEF,KAAK,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpC,KAAK,eAAe,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE5C,KAAK,sBAAsB,GAAG;IAC5B,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,UAAU,CAAC;CACxB,CAAC;AA4BF,eAAO,MAAM,uBAAuB,QAAkB,CAAC;AAoFvD,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI;oBAe7B,MAAM;;EASzB;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,sBAAsB;oBAXhD,MAAM;;EAmBzB;AA6GD,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,OAAO,GAClB,OAAO,CAAC,aAAa,CAAC,CA2CxB"}
1	+ {"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGzE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAoBzD,OAAO,KAAK,EAAC,SAAS,EAAE,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAaxE,OAAO,EAEL,KAAK,UAAU,EAGhB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAU1D,OAAO,EAML,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC/B,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9C;;;;;;OAMG;IACH,MAAM,CAAC,EACH;QACE,6DAA6D;QAC7D,UAAU,EAAE,MAAM,CAAC;QACnB;;;WAGG;QACH,eAAe,EAAE,MAAM,CAAC;KACzB,GACD,SAAS,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,EAAC,QAAQ,EAAC,GAAE,cAAiC,GAC5C,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAuRnC;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,aAAa,EACtB,WAAW,CAAC,EAAE,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,GACjD,OAAO,CAAC,IAAI,CAAC,CA+Bf;AAuKD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE;IAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAAC,OAAO,EAAE,SAAS,EAAE,CAAA;CAAC,EAC/D,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GACvC,IAAI,CAgFN;AAMD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAMpC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAwBF;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,GACb,MAAM,EAAE,CAKV;AAED,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,EACd,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,EAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,EACpC,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,GACjC,kBAAkB,CA8BpB;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,cAAc,CAAC;CACxB,CAAC;AAEF,KAAK,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpC,KAAK,eAAe,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE5C,KAAK,sBAAsB,GAAG;IAC5B,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,UAAU,CAAC;CACxB,CAAC;AA4BF,eAAO,MAAM,uBAAuB,QAAkB,CAAC;AAoFvD,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI;oBAe7B,MAAM;;EASzB;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,sBAAsB;oBAXhD,MAAM;;EAmBzB;AA6GD,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,OAAO,GAClB,OAAO,CAAC,aAAa,CAAC,CA2CxB"}