@rocicorp/zero 1.11.0-canary.40 → 1.11.0-canary.41

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAajF,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EAEvB,MAAM,sBAAsB,CAAC;AAY9B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAGL,KAAK,oCAAoC,EAC1C,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAIL,KAAK,uBAAuB,EAC7B,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAKrB,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9D;;;;OAIG;IACH,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,8DAA8D;IAC9D,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,MAAM,uBAAuB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;CACnE,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C;;;;OAIG;IACH,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;;;;;;;;OASG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5D;;;;;;OAMG;IACH,+BAA+B,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrD;;;;OAIG;IACH,uCAAuC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7D,aAAa,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD;;;OAGG;IACH,qBAAqB,CAAC,EAClB,IAAI,CACF,4BAA4B,EAC5B,UAAU,GAAG,YAAY,GAAG,WAAW,CACxC,GACD,SAAS,CAAC;IACd;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oCAAoC,GAAG,SAAS,CAAC;IACxE;;;;;;OAMG;IACH,sBAAsB,CAAC,EACnB,CAAC,6BAA6B,GAAG;QAAC,WAAW,EAAE,MAAM,CAAA;KAAC,CAAC,GACvD,SAAS,CAAC;IACd,+EAA+E;IAC/E,oBAAoB,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;CAChE,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CAiDhC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAajF,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EAEvB,MAAM,sBAAsB,CAAC;AAa9B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAGL,KAAK,oCAAoC,EAC1C,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAIL,KAAK,uBAAuB,EAC7B,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAKrB,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9D;;;;OAIG;IACH,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,8DAA8D;IAC9D,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,MAAM,uBAAuB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;CACnE,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C;;;;OAIG;IACH,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;;;;;;;;OASG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5D;;;;;;OAMG;IACH,+BAA+B,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrD;;;;OAIG;IACH,uCAAuC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7D,aAAa,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD;;;OAGG;IACH,qBAAqB,CAAC,EAClB,IAAI,CACF,4BAA4B,EAC5B,UAAU,GAAG,YAAY,GAAG,WAAW,CACxC,GACD,SAAS,CAAC;IACd;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oCAAoC,GAAG,SAAS,CAAC;IACxE;;;;;;OAMG;IACH,sBAAsB,CAAC,EACnB,CAAC,6BAA6B,GAAG;QAAC,WAAW,EAAE,MAAM,CAAA;KAAC,CAAC,GACvD,SAAS,CAAC;IACd,+EAA+E;IAC/E,oBAAoB,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;CAChE,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CAmDhC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -8,7 +8,7 @@ import { DEFAULT_MAX_RETRY_DELAY_MS, RunningState, UnrecoverableError } from "..
 import { min } from "../../types/lexi-version.js";
 import { publishCriticalEvent } from "../../observability/events.js";
 import { Subscription } from "../../types/subscription.js";
-import { AutoResetSignal, ensureReplicationConfig, markResetRequired } from "./schema/tables.js";
+import { AutoResetSignal, checkReplicationConfig, markResetRequired } from "./schema/tables.js";
 import { publishReplicationError, replicationStatusError } from "../replicator/replication-status.js";
 import "./change-streamer.js";
 import { BackfillStateTracker } from "./backfill-state-tracker.js";
@@ -36,13 +36,12 @@ import { getDefaultHighWaterMark } from "node:stream";
 * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
 */
 async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, backupConfig, purgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
-	await ensureReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, purgeLock ?? void 0, setTimeoutFn, opts.pgChangeLogEnabled);
+	if (opts.pgChangeLogEnabled) await checkReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, opts.pgChangeLogBehindSlot);
 	const changeDBProvider = (applicationName, max) => postgres(defu({
 		max,
 		connection: { ["application_name"]: applicationName }
 	}, changeDB.options));
-	const { replicaVersion } = subscriptionState;
-	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, replicaVersion, changeSource, replicationStatusPublisher, backupConfig, purgeLock, autoReset, opts, setTimeoutFn);
+	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, subscriptionState, changeSource, replicationStatusPublisher, backupConfig, purgeLock, autoReset, opts, setTimeoutFn);
 }
 var REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5e3;
 var DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS = 6e4;
@@ -193,6 +192,7 @@ var ChangeStreamerImpl = class {
 	#source;
 	#storer;
 	#pgChangeLogEnabled;
+	#pgChangeLogBehindSlot;
 	#forwarder;
 	#reservations;
 	#restoreAdmission;
@@ -239,21 +239,23 @@ var ChangeStreamerImpl = class {
 	#lastForwardedCommitWatermark;
 	#transactionForwardStartedAt;
 	#currentTransactionCompletion;
-	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, replicaVersion, source, replicationStatusPublisher, backupConfig, initialPurgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
+	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, subscriptionState, source, replicationStatusPublisher, backupConfig, initialPurgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
 		this.id = `change-streamer`;
 		this.#lc = lc.withContext("component", "change-streamer");
 		this.#backfills = new BackfillStateTracker(this.#lc);
 		this.#shard = shard;
 		this.#changeDBProvider = changeDBProvider;
+		const { replicaVersion } = subscriptionState;
 		this.#replicaVersion = replicaVersion;
 		this.#source = source;
 		this.#pgChangeLogEnabled = opts.pgChangeLogEnabled;
+		this.#pgChangeLogBehindSlot = opts.pgChangeLogBehindSlot;
 		if (!this.#pgChangeLogEnabled) {
 			assert(opts.sqliteChangeLogWriter && opts.sqliteChangeLogPurge && opts.sqliteCatchup && opts.sqliteChangeLogServe?.readPercent === 100 && opts.sqliteChangeLogServe.coldReadPercent === 100 && backupConfig?.litestreamVersion === "v5", "disabling the PG change log requires SQLite serving at 100 percent and a v5 backup");
 			assert(opts.sqliteChangeLogCompare === void 0, "SQLite change-log comparison requires the PG change log");
 		}
 		assert(!(this.#pgChangeLogEnabled && opts.pgChangeLogBehindSlot && !backupConfig?.replicaFile), "the PG change log is behind the replication slot, but there is no restored replica to re-initialize it from");
-		this.#storer = new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, replicaVersion, (consumed) => this.#acker.trackPgChangeLog(consumed[2].watermark), (err) => this.stop(err), opts);
+		this.#storer = new Storer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDBProvider, subscriptionState, (consumed) => this.#acker.trackPgChangeLog(consumed[2].watermark), (err) => this.stop(err), opts);
 		this.#forwarder = new Forwarder(lc, {
 			flowControlConsensusTimeoutProportion: opts.flowControlConsensusTimeoutProportion,
 			flowControlSlowSubscriberGracePeriodMs: opts.flowControlSlowSubscriberGracePeriodMs
@@ -337,17 +339,23 @@ var ChangeStreamerImpl = class {
 		const lagReportInit = await this.#source.startLagReporter();
 		this.#lagReportingDisabled.resolve(!lagReportInit);
 		if (lagReportInit) this.#latestLagReportCommitTimeMs = lagReportInit.firstCommitTimeMs;
-		if (this.#pgChangeLogEnabled) this.#storer.assumeOwnershipAtHead({
-			purgeLock: this.#purgeLock,
-			requestRestart: (reason) => this.#backfills.requestRestart(reason)
-		});
-		this.#purgeLock = null;
+		let takeoverPending = this.#pgChangeLogEnabled;
 		const flushBytesThreshold = getDefaultHighWaterMark(false);
 		while (this.#state.shouldRun()) {
 			let err;
 			let watermark = null;
 			let unflushedBytes = 0;
 			try {
+				if (takeoverPending) {
+					await this.#storer.assumeOwnershipAtHead({
+						purgeLock: this.#purgeLock,
+						requestRestart: (reason) => this.#backfills.requestRestart(reason),
+						autoReset: this.#autoReset,
+						pgChangeLogBehindSlot: this.#pgChangeLogBehindSlot
+					});
+					takeoverPending = false;
+					this.#purgeLock = null;
+				}
 				const { lastWatermark, backfillRequests: logBackfillRequests } = await this.#initializer.initialize();
 				const backfillRequests = this.#backfills.startStream(this.#pgChangeLogEnabled ? logBackfillRequests : []);
 				this.#lastForwardedCommitWatermark = lastWatermark;
@@ -448,7 +456,7 @@ var ChangeStreamerImpl = class {
 		const { tag } = msg;
 		switch (tag) {
 			case "reset-required":
-				await markResetRequired(this.#changeDBProvider("change-streamer-reset", 1), this.#shard);
+				if (this.#pgChangeLogEnabled) await markResetRequired(this.#changeDBProvider("change-streamer-reset", 1), this.#shard);
 				await publishReplicationError(this.#lc, "Replicating", msg.message ?? "Resync required", msg.errorDetails);
 				if (this.#autoReset) {
 					this.#lc.warn?.("shutting down for auto-reset");