@rocicorp/zero 1.11.0-canary.0 → 1.11.0-canary.10

package/out/zero-cache/src/services/change-streamer/forwarder.js

@@ -2,6 +2,7 @@ import { joinIterables, wrapIterable } from "../../../../shared/src/iterables.js
 import { getOrCreateCounter, getOrCreateGauge, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
 import { Broadcast } from "./broadcast.js";
 //#region ../zero-cache/src/services/change-streamer/forwarder.ts
+var FORWARD_BATCH_SIZE = 64;
 var Forwarder = class {
 	#lc;
 	#setTimeout;
@@ -13,6 +14,9 @@ var Forwarder = class {
 	#flowControlWaits = getOrCreateCounter("replication", "flow_control.waits", "Flow-control checkpoints completed, labeled by release mode.");
 	#flowControlWaitTime = getOrCreateLatencyHistogram("replication", "flow_control.wait_duration", "Time replication waits at flow-control checkpoints.");
 	#inTransaction = false;
+	#pending = [];
+	#flushScheduled = false;
+	#flushImmediateId = void 0;
 	#currentBroadcast;
 	#progressMonitor;
 	#backupLagEventCount = 0;
@@ -79,6 +83,7 @@ var Forwarder = class {
 	}
 	stopProgressMonitor() {
 		clearInterval(this.#progressMonitor);
+		this.#flushPendingWithoutTracking();
 	}
 	/**
 	* `add()` is called in lock step with `Storer.catchup()` so that the
@@ -86,10 +91,12 @@ var Forwarder = class {
 	* currently being streamed.
 	*/
 	add(sub) {
+		this.#flushPendingWithoutTracking();
 		if (this.#inTransaction) this.#queued.add(sub);
 		else this.#active.add(sub);
 	}
 	remove(sub) {
+		this.#flushPendingWithoutTracking();
 		this.#active.delete(sub);
 		this.#queued.delete(sub);
 		sub.close();
@@ -99,15 +106,22 @@ var Forwarder = class {
 	* two components have an equivalent interpretation of whether a Transaction is
 	* currently being streamed.
 	*
-	* This version of forward is fire-and-forget, with no flow control. The
-	* change-streamer should call and await {@link forwardWithFlowControl()}
-	* occasionally to avoid memory blowup.
+	* This version of forward buffers changes across the current event loop turn
+	* and dispatches them in batches. The change-streamer should call and await
+	* {@link forwardWithFlowControl()} occasionally to avoid memory blowup.
 	*/
 	forward(entry) {
-		Broadcast.withoutTracking(this.#active.values(), entry);
+		this.#pending.push(entry);
+		if (this.#queued.size > 0 && (entry[1] === "commit" || entry[1] === "rollback")) this.#flushPendingWithoutTracking();
 		this.#updateActiveSubscribers(entry[1]);
+		if (this.#pending.length >= FORWARD_BATCH_SIZE) this.#flushPendingWithoutTracking();
+		else if (!this.#flushScheduled && this.#pending.length > 0) {
+			this.#flushScheduled = true;
+			this.#flushImmediateId = setImmediate(() => this.#flushPendingWithoutTracking());
+		}
 	}
 	sendStatus(status) {
+		this.#flushPendingWithoutTracking();
 		for (const sub of this.#active.values()) sub.sendStatus(status);
 	}
 	/**
@@ -115,8 +129,10 @@ var Forwarder = class {
 	* Promise that resolves when replication should continue.
 	*/
 	async forwardWithFlowControl(entry) {
+		this.#pending.push(entry);
+		const batch = this.#drainPending();
 		const start = performance.now();
-		const broadcast = new Broadcast(this.#lc, this.#active.values(), entry, this.#earlyReleaseOptions);
+		const broadcast = new Broadcast(this.#lc, this.#active.values(), batch, this.#earlyReleaseOptions);
 		this.#updateActiveSubscribers(entry[1]);
 		this.#currentBroadcast = broadcast;
 		try {
@@ -128,6 +144,21 @@ var Forwarder = class {
 			if (this.#currentBroadcast === broadcast) this.#currentBroadcast = void 0;
 		}
 	}
+	#drainPending() {
+		if (this.#flushImmediateId !== void 0) {
+			clearImmediate(this.#flushImmediateId);
+			this.#flushImmediateId = void 0;
+		}
+		this.#flushScheduled = false;
+		if (this.#pending.length === 0) return [];
+		const batch = this.#pending;
+		this.#pending = [];
+		return batch;
+	}
+	#flushPendingWithoutTracking() {
+		const batch = this.#drainPending();
+		if (batch.length > 0) Broadcast.withoutTracking(this.#active.values(), batch);
+	}
 	#subscriberStats() {
 		let pendingMessages = 0;
 		let backlogMessages = 0;

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#backupLagEventCount","#lastBackupLagWarnMs","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #earlyReleaseOptions: EarlyReleaseOptions | undefined;\n  readonly #flowControlSlowSubscriberGracePeriodMs: number | undefined;\n\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  // The backup-replicator is never disconnected as a laggard, so its lag\n  // detections are logged (rather than acted upon). Track the running total and\n  // throttle the warning to at most once per grace period to avoid spamming the\n  // log for the duration of a slow catchup.\n  #backupLagEventCount = 0;\n  #lastBackupLagWarnMs: number | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {\n      flowControlConsensusTimeoutProportion: 2.0,\n    },\n  ) {\n    this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    this.#earlyReleaseOptions =\n      opts.flowControlConsensusTimeoutProportion < 0\n        ? undefined\n        : {\n            consensusTimeoutProportion:\n              opts.flowControlConsensusTimeoutProportion,\n            setTimeoutFn: this.#setTimeout,\n            clearTimeoutFn: this.#clearTimeout,\n          };\n    this.#flowControlSlowSubscriberGracePeriodMs =\n      opts.flowControlSlowSubscriberGracePeriodMs;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(\n      () => this.checkSubscriberProgress(performance.now()),\n      1000,\n    );\n  }\n\n  /**\n   * Runs one progress-monitor tick: samples each active subscriber's change\n   * rate, logs slow broadcasts, and disconnects subscribers that have been\n   * continuously lagging for longer than the configured grace period.\n   *\n   * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n   * than private) so tests can drive a single tick with an explicit `now`.\n   */\n  checkSubscriberProgress(now: number) {\n    let timedOut: Map<Subscriber, SubscriberStats> | undefined;\n    let slowestOnTime: SubscriberStats | undefined;\n\n    for (const sub of this.#active) {\n      const stats = sub.sampleProcessRate(now).getStats();\n      if (stats.missedLastTimeout) {\n        (timedOut ??= new Map()).set(sub, stats);\n      } else if (\n        stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n      ) {\n        slowestOnTime = stats;\n      }\n    }\n    this.#currentBroadcast?.logSlowRequests(now);\n\n    if (\n      this.#flowControlSlowSubscriberGracePeriodMs &&\n      timedOut?.size &&\n      slowestOnTime\n    ) {\n      for (const [sub, stats] of timedOut.entries()) {\n        const rate =\n          stats.processRate < slowestOnTime.processRate\n            ? 'lagging'\n            : 'catching-up';\n        const laggingDuration = sub.reportChangeRate(now, rate);\n        if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n          if (sub.mode === 'backup') {\n            // The backup-replicator can only be detected as a laggard during\n            // initial catchup; after that it is required for consensus. If it\n            // triggers laggard detection in the former case, just log and\n            // continue; as disconnecting it would otherwise kill the\n            // change-streamer task altogether.\n            this.#backupLagEventCount++;\n            const grace = this.#flowControlSlowSubscriberGracePeriodMs;\n            if (\n              this.#lastBackupLagWarnMs === undefined ||\n              now - this.#lastBackupLagWarnMs >= grace\n            ) {\n              this.#lastBackupLagWarnMs = now;\n              this.#lc.warn?.(\n                `backup subscriber ${sub.id} has lagged for ${laggingDuration}ms ` +\n                  `(${this.#backupLagEventCount} lag events so far). ` +\n                  `this is only expected during initial catchup`,\n                {stats},\n              );\n            }\n          } else {\n            this.#lc.warn?.(\n              `serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n              {stats},\n            );\n            sub.close(\n              StreamTooFarBehind,\n              `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n            );\n          }\n        }\n      }\n    }\n  }\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const start = performance.now();\n    const broadcast = new Broadcast(\n      this.#lc,\n      this.#active.values(),\n      entry,\n      this.#earlyReleaseOptions,\n    );\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAoBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAMA,uBAAuB;CACvB;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAC1B,IAAI,IAAI,SAAS,UAAU;IAMzB,KAAKM;IACL,MAAM,QAAQ,KAAKN;IACnB,IACE,KAAKO,yBAAyB,KAAA,KAC9B,MAAM,KAAKA,wBAAwB,OACnC;KACA,KAAKA,uBAAuB;KAC5B,KAAKb,IAAI,OACP,qBAAqB,IAAI,GAAG,kBAAkB,gBAAgB,MACxD,KAAKY,qBAAqB,oEAEhC,EAAC,MAAK,CACR;IACF;GACF,OAAO;IACL,KAAKZ,IAAI,OACP,sBAAsB,IAAI,GAAG,kBAAkB,gBAAgB,oBAC/D,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EAEJ;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKI,gBACP,KAAKV,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKY,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKZ,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKH,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKU,yBAAyB,MAAM,EAAE;EAGtC,KAAKJ,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKR,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKU,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKV,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}
+{"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#backupLagEventCount","#lastBackupLagWarnMs","#flushPendingWithoutTracking","#inTransaction","#pending","#updateActiveSubscribers","#flushScheduled","#flushImmediateId","#drainPending"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nconst FORWARD_BATCH_SIZE = 64;\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #earlyReleaseOptions: EarlyReleaseOptions | undefined;\n  readonly #flowControlSlowSubscriberGracePeriodMs: number | undefined;\n\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #pending: WatermarkedChange[] = [];\n  #flushScheduled = false;\n  #flushImmediateId: NodeJS.Immediate | undefined = undefined;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  // The backup-replicator is never disconnected as a laggard, so its lag\n  // detections are logged (rather than acted upon). Track the running total and\n  // throttle the warning to at most once per grace period to avoid spamming the\n  // log for the duration of a slow catchup.\n  #backupLagEventCount = 0;\n  #lastBackupLagWarnMs: number | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {\n      flowControlConsensusTimeoutProportion: 2.0,\n    },\n  ) {\n    this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    this.#earlyReleaseOptions =\n      opts.flowControlConsensusTimeoutProportion < 0\n        ? undefined\n        : {\n            consensusTimeoutProportion:\n              opts.flowControlConsensusTimeoutProportion,\n            setTimeoutFn: this.#setTimeout,\n            clearTimeoutFn: this.#clearTimeout,\n          };\n    this.#flowControlSlowSubscriberGracePeriodMs =\n      opts.flowControlSlowSubscriberGracePeriodMs;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(\n      () => this.checkSubscriberProgress(performance.now()),\n      1000,\n    );\n  }\n\n  /**\n   * Runs one progress-monitor tick: samples each active subscriber's change\n   * rate, logs slow broadcasts, and disconnects subscribers that have been\n   * continuously lagging for longer than the configured grace period.\n   *\n   * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n   * than private) so tests can drive a single tick with an explicit `now`.\n   */\n  checkSubscriberProgress(now: number) {\n    let timedOut: Map<Subscriber, SubscriberStats> | undefined;\n    let slowestOnTime: SubscriberStats | undefined;\n\n    for (const sub of this.#active) {\n      const stats = sub.sampleProcessRate(now).getStats();\n      if (stats.missedLastTimeout) {\n        (timedOut ??= new Map()).set(sub, stats);\n      } else if (\n        stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n      ) {\n        slowestOnTime = stats;\n      }\n    }\n    this.#currentBroadcast?.logSlowRequests(now);\n\n    if (\n      this.#flowControlSlowSubscriberGracePeriodMs &&\n      timedOut?.size &&\n      slowestOnTime\n    ) {\n      for (const [sub, stats] of timedOut.entries()) {\n        const rate =\n          stats.processRate < slowestOnTime.processRate\n            ? 'lagging'\n            : 'catching-up';\n        const laggingDuration = sub.reportChangeRate(now, rate);\n        if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n          if (sub.mode === 'backup') {\n            // The backup-replicator can only be detected as a laggard during\n            // initial catchup; after that it is required for consensus. If it\n            // triggers laggard detection in the former case, just log and\n            // continue; as disconnecting it would otherwise kill the\n            // change-streamer task altogether.\n            this.#backupLagEventCount++;\n            const grace = this.#flowControlSlowSubscriberGracePeriodMs;\n            if (\n              this.#lastBackupLagWarnMs === undefined ||\n              now - this.#lastBackupLagWarnMs >= grace\n            ) {\n              this.#lastBackupLagWarnMs = now;\n              this.#lc.warn?.(\n                `backup subscriber ${sub.id} has lagged for ${laggingDuration}ms ` +\n                  `(${this.#backupLagEventCount} lag events so far). ` +\n                  `this is only expected during initial catchup`,\n                {stats},\n              );\n            }\n          } else {\n            this.#lc.warn?.(\n              `serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n              {stats},\n            );\n            sub.close(\n              StreamTooFarBehind,\n              `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n            );\n          }\n        }\n      }\n    }\n  }\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#flushPendingWithoutTracking();\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    this.#flushPendingWithoutTracking();\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#flushPendingWithoutTracking();\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward buffers changes across the current event loop turn\n   * and dispatches them in batches. The change-streamer should call and await\n   * {@link forwardWithFlowControl()} occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    this.#pending.push(entry);\n\n    if (\n      this.#queued.size > 0 &&\n      (entry[1] === 'commit' || entry[1] === 'rollback')\n    ) {\n      this.#flushPendingWithoutTracking();\n    }\n    this.#updateActiveSubscribers(entry[1]);\n\n    if (this.#pending.length >= FORWARD_BATCH_SIZE) {\n      this.#flushPendingWithoutTracking();\n    } else if (!this.#flushScheduled && this.#pending.length > 0) {\n      this.#flushScheduled = true;\n      this.#flushImmediateId = setImmediate(() =>\n        this.#flushPendingWithoutTracking(),\n      );\n    }\n  }\n\n  sendStatus(status: Status) {\n    this.#flushPendingWithoutTracking();\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    this.#pending.push(entry);\n\n    const batch = this.#drainPending();\n    const start = performance.now();\n    const broadcast = new Broadcast(\n      this.#lc,\n      this.#active.values(),\n      batch,\n      this.#earlyReleaseOptions,\n    );\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #drainPending(): WatermarkedChange[] {\n    if (this.#flushImmediateId !== undefined) {\n      clearImmediate(this.#flushImmediateId);\n      this.#flushImmediateId = undefined;\n    }\n    this.#flushScheduled = false;\n    if (this.#pending.length === 0) {\n      return [];\n    }\n    const batch = this.#pending;\n    this.#pending = [];\n    return batch;\n  }\n\n  #flushPendingWithoutTracking() {\n    const batch = this.#drainPending();\n    if (batch.length > 0) {\n      Broadcast.withoutTracking(this.#active.values(), batch);\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAoBA,IAAM,qBAAqB;AAE3B,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB,WAAgC,CAAC;CACjC,kBAAkB;CAClB,oBAAkD,KAAA;CAElD;CACA;CAMA,uBAAuB;CACvB;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAC1B,IAAI,IAAI,SAAS,UAAU;IAMzB,KAAKM;IACL,MAAM,QAAQ,KAAKN;IACnB,IACE,KAAKO,yBAAyB,KAAA,KAC9B,MAAM,KAAKA,wBAAwB,OACnC;KACA,KAAKA,uBAAuB;KAC5B,KAAKb,IAAI,OACP,qBAAqB,IAAI,GAAG,kBAAkB,gBAAgB,MACxD,KAAKY,qBAAqB,oEAEhC,EAAC,MAAK,CACR;IACF;GACF,OAAO;IACL,KAAKZ,IAAI,OACP,sBAAsB,IAAI,GAAG,kBAAkB,gBAAgB,oBAC/D,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EAEJ;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;EACnC,KAAKI,6BAA6B;CACpC;;;;;;CAOA,IAAI,KAAiB;EACnB,KAAKA,6BAA6B;EAClC,IAAI,KAAKC,gBACP,KAAKX,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKW,6BAA6B;EAClC,KAAKX,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,KAAKY,SAAS,KAAK,KAAK;EAExB,IACE,KAAKZ,QAAQ,OAAO,MACnB,MAAM,OAAO,YAAY,MAAM,OAAO,aAEvC,KAAKU,6BAA6B;EAEpC,KAAKG,yBAAyB,MAAM,EAAE;EAEtC,IAAI,KAAKD,SAAS,UAAU,oBAC1B,KAAKF,6BAA6B;OAC7B,IAAI,CAAC,KAAKI,mBAAmB,KAAKF,SAAS,SAAS,GAAG;GAC5D,KAAKE,kBAAkB;GACvB,KAAKC,oBAAoB,mBACvB,KAAKL,6BAA6B,CACpC;EACF;CACF;CAEA,WAAW,QAAgB;EACzB,KAAKA,6BAA6B;EAClC,KAAK,MAAM,OAAO,KAAKX,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,KAAKa,SAAS,KAAK,KAAK;EAExB,MAAM,QAAQ,KAAKI,cAAc;EACjC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKpB,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKY,yBAAyB,MAAM,EAAE;EAGtC,KAAKN,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,gBAAqC;EACnC,IAAI,KAAKQ,sBAAsB,KAAA,GAAW;GACxC,eAAe,KAAKA,iBAAiB;GACrC,KAAKA,oBAAoB,KAAA;EAC3B;EACA,KAAKD,kBAAkB;EACvB,IAAI,KAAKF,SAAS,WAAW,GAC3B,OAAO,CAAC;EAEV,MAAM,QAAQ,KAAKA;EACnB,KAAKA,WAAW,CAAC;EACjB,OAAO;CACT;CAEA,+BAA+B;EAC7B,MAAM,QAAQ,KAAKI,cAAc;EACjC,IAAI,MAAM,SAAS,GACjB,UAAU,gBAAgB,KAAKjB,QAAQ,OAAO,GAAG,KAAK;CAE1D;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKA,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKW,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKX,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts

@@ -48,7 +48,7 @@ interface PurgeLock {
 }
 export declare function setupCDCTables(lc: LogContext, db: postgres.TransactionSql, shard: ShardID): Promise<void>;
 export declare function markResetRequired(sql: PostgresDB, shard: ShardID): Promise<void>;
-export declare function ensureReplicationConfig(lc: LogContext, db: PostgresDB, subscriptionState: Pick<SubscriptionState, 'publications' | 'replicaVersion' | 'watermark'>, shard: ShardID, autoReset: boolean, purgeLock?: PurgeLock, setTimeoutFn?: typeof setTimeout): Promise<void>;
+export declare function ensureReplicationConfig(lc: LogContext, db: PostgresDB, subscriptionState: Pick<SubscriptionState, 'publications' | 'replicaVersion' | 'watermark'>, shard: ShardID, autoReset: boolean, purgeLock?: PurgeLock, setTimeoutFn?: typeof setTimeout, pgChangeLogEnabled?: boolean): Promise<void>;
 export declare const CHANGE_STREAMER_APP_NAME = "zero-change-streamer";
 export declare class AutoResetSignal extends AbortError {
     readonly name = "AutoResetSignal";

package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EACV,UAAU,EACV,MAAM,EACN,aAAa,EACd,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAmBF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAaF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,UAiBlD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAYF,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,IAAI,CACrB,iBAAiB,EACjB,cAAc,GAAG,gBAAgB,GAAG,WAAW,CAChD,EACD,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,EAClB,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,iBA4H7C;AAMD,eAAO,MAAM,wBAAwB,yBAAyB,CAAC;AAE/D,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,iBA4Cf"}
+{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EACV,UAAU,EACV,MAAM,EACN,aAAa,EACd,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAmBF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAaF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,UAiBlD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAYF,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,IAAI,CACrB,iBAAiB,EACjB,cAAc,GAAG,gBAAgB,GAAG,WAAW,CAChD,EACD,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,EAClB,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,EAC5C,kBAAkB,UAAO,iBAgI1B;AAMD,eAAO,MAAM,wBAAwB,yBAAyB,CAAC;AAE/D,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,iBA4Cf"}

package/out/zero-cache/src/services/change-streamer/schema/tables.js

@@ -77,7 +77,7 @@ async function markResetRequired(sql, shard) {
   UPDATE ${sql(cdcSchema(shard))}."replicationConfig"
     SET "resetRequired" = true`;
 }
-async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoReset, purgeLock, setTimeoutFn = setTimeout) {
+async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoReset, purgeLock, setTimeoutFn = setTimeout, pgChangeLogEnabled = true) {
 	const { publications, replicaVersion, watermark } = subscriptionState;
 	const replicaConfig = {
 		publications,
@@ -102,12 +102,12 @@ async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoRes
 				lc.info?.(`Data in cdc tables @${replicaVersion} is incompatible with replica @${replicaConfig.replicaVersion}. Clearing tables.`);
 				purgeLock?.release();
 				needsTruncate = true;
-				stmts.push(sql`TRUNCATE TABLE ${sql(schema)}."replicationState"`, sql`TRUNCATE TABLE ${sql(schema)}."changeLog"`, sql`TRUNCATE TABLE ${sql(schema)}."replicationConfig"`, sql`TRUNCATE TABLE ${sql(schema)}."tableMetadata"`, sql`TRUNCATE TABLE ${sql(schema)}."backfilling"`);
+				stmts.push(sql`TRUNCATE TABLE ${sql(schema)}."replicationState"`, ...pgChangeLogEnabled ? [sql`TRUNCATE TABLE ${sql(schema)}."changeLog"`] : [], sql`TRUNCATE TABLE ${sql(schema)}."replicationConfig"`, sql`TRUNCATE TABLE ${sql(schema)}."tableMetadata"`, sql`TRUNCATE TABLE ${sql(schema)}."backfilling"`);
 			}
 		}
 		if (results.length === 0 || needsTruncate) {
 			const watermark = replicaConfig.replicaVersion;
-			const initialTx = [{
+			const initialTx = pgChangeLogEnabled ? [{
 				watermark,
 				pos: 0,
 				change: { tag: "begin" }
@@ -115,7 +115,7 @@ async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoRes
 				watermark,
 				pos: 1,
 				change: { tag: "commit" }
-			}];
+			}] : [];
 			stmts.push(sql`INSERT INTO ${sql(schema)}."replicationConfig" ${sql(replicaConfig)}`, sql`INSERT INTO ${sql(schema)}."replicationState"  ${sql(replicationState)} 
               ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`, ...initialTx.map((change) => sql`INSERT INTO ${sql(schema)}."changeLog" ${sql(change)}`));
 			if (needsTruncate) {

package/out/zero-cache/src/services/change-streamer/schema/tables.js.map

@@ -1 +1 @@
-{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: Pick<\n    SubscriptionState,\n    'publications' | 'replicaVersion' | 'watermark'\n  >,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n) {\n  const {publications, replicaVersion, watermark} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const stmts: PendingQuery<Row[]>[] = [];\n    let needsTruncate = false;\n    const results = await sql<\n      {\n        replicaVersion: string;\n        publications: string[];\n        resetRequired: boolean | null;\n      }[]\n    > /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(schema)}.\"replicationConfig\"`;\n\n    if (results.length) {\n      const {replicaVersion, publications} = results[0];\n      if (\n        replicaVersion !== replicaConfig.replicaVersion ||\n        !equals(new Set(publications), new Set(replicaConfig.publications))\n      ) {\n        if (replicaConfig.replicaVersion !== watermark) {\n          throw new AutoResetSignal(\n            `Cannot reset change db@${replicaVersion} to ` +\n              `service replica@${replicaConfig.replicaVersion} ` +\n              `from watermark ${watermark}`,\n          );\n        }\n        lc.info?.(\n          `Data in cdc tables @${replicaVersion} is incompatible ` +\n            `with replica @${replicaConfig.replicaVersion}. Clearing tables.`,\n        );\n        // Release any purge lock held by the caller; the changeLog is\n        // incompatible with the replica and needs to be truncated.\n        void purgeLock?.release();\n\n        // Note: The order of TRUNCATE matters. The replicationState table is\n        //       truncated before the changeLog table, in order to acquire\n        //       (exclusive) locks on the tables in the same order that the\n        //       storer.ts acquires locks when writing the changeLog, namely:\n        //\n        // 1. SELECT ... FROM replicationState FOR UPDATE;\n        // 2. INSERT INTO changeLog ...;\n        //    ...\n        // n. UPDATE replicationState ...;\n        needsTruncate = true;\n        stmts.push(\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n        );\n      }\n    }\n    // Initialize (or re-initialize TRUNCATED) tables\n    if (results.length === 0 || needsTruncate) {\n      // The storer uses the earliest changeLog entry as the safe watermark\n      // from which subscribers can be resumed. These initial entries ensure\n      // that subscribers can start from a freshly synced replica, even if\n      // new changes have been replicated and not purged from the changeLog.\n      //\n      // TODO: Replace this with an explicit `firstWatermark` column in the\n      //       change db.\n      const watermark = replicaConfig.replicaVersion;\n      const initialTx: FullChangeLogEntry[] = [\n        {watermark, pos: 0, change: {tag: 'begin'}},\n        {watermark, pos: 1, change: {tag: 'commit'}},\n      ];\n\n      stmts.push(\n        sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n        sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n              ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n        ...initialTx.map(\n          change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n        ),\n      );\n\n      if (needsTruncate) {\n        // The TRUNCATE statements require ACCESS EXCLUSIVE locks, which may\n        // be blocked by old storer catchup reads. Race against a timeout\n        // that terminates the blocking backends if the TRUNCATE takes too\n        // long.\n        const timer = setTimeoutFn(async () => {\n          lc.info?.(\n            'ensureReplicationConfig blocked, terminating lock holders',\n          );\n          await terminateChangeDBLockHolders(lc, db, shard);\n        }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n        try {\n          return await Promise.all(stmts);\n        } finally {\n          clearTimeout(timer);\n        }\n      }\n      return Promise.all(stmts);\n    }\n\n    const {resetRequired} = results[0];\n    if (resetRequired) {\n      if (autoReset) {\n        throw new AutoResetSignal('reset required by replication stream');\n      }\n      lc.error?.(\n        '\\n\\n\\n' +\n          'Reset required but --auto-reset is not enabled.\\n' +\n          'This can happen for upstream databases that do not support event triggers.\\n' +\n          'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n          '\\n\\n\\n',\n      );\n    }\n\n    return [];\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nexport async function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,sBACvC,GAAG;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;AAEA,eAAsB,wBACpB,IACA,IACA,mBAIA,OACA,WACA,WACA,eAAkC,YAClC;CACA,MAAM,EAAC,cAAc,gBAAgB,cAAa;CAClD,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,QAA+B,CAAC;EACtC,IAAI,gBAAgB;EACpB,MAAM,UAAU,MAAM,GAMZ;;aAED,IAAI,MAAM,EAAE;EAErB,IAAI,QAAQ,QAAQ;GAClB,MAAM,EAAC,gBAAgB,iBAAgB,QAAQ;GAC/C,IACE,mBAAmB,cAAc,kBACjC,CAAC,OAAO,IAAI,IAAI,YAAY,GAAG,IAAI,IAAI,cAAc,YAAY,CAAC,GAClE;IACA,IAAI,cAAc,mBAAmB,WACnC,MAAM,IAAI,gBACR,0BAA0B,eAAe,sBACpB,cAAc,eAAe,kBAC9B,WACtB;IAEF,GAAG,OACD,uBAAuB,eAAe,iCACnB,cAAc,eAAe,mBAClD;IAGA,WAAgB,QAAQ;IAWxB,gBAAgB;IAChB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;GACF;EACF;EAEA,IAAI,QAAQ,WAAW,KAAK,eAAe;GAQzC,MAAM,YAAY,cAAc;GAChC,MAAM,YAAkC,CACtC;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,QAAO;GAAC,GAC1C;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,SAAQ;GAAC,CAC7C;GAEA,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;iDAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;GAEA,IAAI,eAAe;IAKjB,MAAM,QAAQ,aAAa,YAAY;KACrC,GAAG,OACD,2DACF;KACA,MAAM,6BAA6B,IAAI,IAAI,KAAK;IAClD,GAAG,gCAAgC;IACnC,IAAI;KACF,OAAO,MAAM,QAAQ,IAAI,KAAK;IAChC,UAAU;KACR,aAAa,KAAK;IACpB;GACF;GACA,OAAO,QAAQ,IAAI,KAAK;EAC1B;EAEA,MAAM,EAAC,kBAAiB,QAAQ;EAChC,IAAI,eAAe;GACjB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;GAElE,GAAG,QACD,oOAKF;EACF;EAEA,OAAO,CAAC;CACV,CAAC;AACH;AAIA,IAAM,mCAAmC;AAEzC,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}
+{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: Pick<\n    SubscriptionState,\n    'publications' | 'replicaVersion' | 'watermark'\n  >,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n  pgChangeLogEnabled = true,\n) {\n  const {publications, replicaVersion, watermark} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const stmts: PendingQuery<Row[]>[] = [];\n    let needsTruncate = false;\n    const results = await sql<\n      {\n        replicaVersion: string;\n        publications: string[];\n        resetRequired: boolean | null;\n      }[]\n    > /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(schema)}.\"replicationConfig\"`;\n\n    if (results.length) {\n      const {replicaVersion, publications} = results[0];\n      if (\n        replicaVersion !== replicaConfig.replicaVersion ||\n        !equals(new Set(publications), new Set(replicaConfig.publications))\n      ) {\n        if (replicaConfig.replicaVersion !== watermark) {\n          throw new AutoResetSignal(\n            `Cannot reset change db@${replicaVersion} to ` +\n              `service replica@${replicaConfig.replicaVersion} ` +\n              `from watermark ${watermark}`,\n          );\n        }\n        lc.info?.(\n          `Data in cdc tables @${replicaVersion} is incompatible ` +\n            `with replica @${replicaConfig.replicaVersion}. Clearing tables.`,\n        );\n        // Release any purge lock held by the caller; the changeLog is\n        // incompatible with the replica and needs to be truncated.\n        void purgeLock?.release();\n\n        // Note: The order of TRUNCATE matters. The replicationState table is\n        //       truncated before the changeLog table, in order to acquire\n        //       (exclusive) locks on the tables in the same order that the\n        //       storer.ts acquires locks when writing the changeLog, namely:\n        //\n        // 1. SELECT ... FROM replicationState FOR UPDATE;\n        // 2. INSERT INTO changeLog ...;\n        //    ...\n        // n. UPDATE replicationState ...;\n        needsTruncate = true;\n        stmts.push(\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n          ...(pgChangeLogEnabled\n            ? [sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`]\n            : []),\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n        );\n      }\n    }\n    // Initialize (or re-initialize TRUNCATED) tables\n    if (results.length === 0 || needsTruncate) {\n      // When enabled, the PG storer uses the earliest changeLog entry as the\n      // safe watermark from which subscribers can be resumed. These initial\n      // entries ensure that subscribers can start from a freshly synced\n      // replica, even if new changes have been replicated and not purged.\n      //\n      // TODO: Replace this with an explicit `firstWatermark` column in the\n      //       change db.\n      const watermark = replicaConfig.replicaVersion;\n      const initialTx: FullChangeLogEntry[] = pgChangeLogEnabled\n        ? [\n            {watermark, pos: 0, change: {tag: 'begin'}},\n            {watermark, pos: 1, change: {tag: 'commit'}},\n          ]\n        : [];\n\n      stmts.push(\n        sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n        sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n              ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n        ...initialTx.map(\n          change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n        ),\n      );\n\n      if (needsTruncate) {\n        // The TRUNCATE statements require ACCESS EXCLUSIVE locks, which may\n        // be blocked by old storer catchup reads. Race against a timeout\n        // that terminates the blocking backends if the TRUNCATE takes too\n        // long.\n        const timer = setTimeoutFn(async () => {\n          lc.info?.(\n            'ensureReplicationConfig blocked, terminating lock holders',\n          );\n          await terminateChangeDBLockHolders(lc, db, shard);\n        }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n        try {\n          return await Promise.all(stmts);\n        } finally {\n          clearTimeout(timer);\n        }\n      }\n      return Promise.all(stmts);\n    }\n\n    const {resetRequired} = results[0];\n    if (resetRequired) {\n      if (autoReset) {\n        throw new AutoResetSignal('reset required by replication stream');\n      }\n      lc.error?.(\n        '\\n\\n\\n' +\n          'Reset required but --auto-reset is not enabled.\\n' +\n          'This can happen for upstream databases that do not support event triggers.\\n' +\n          'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n          '\\n\\n\\n',\n      );\n    }\n\n    return [];\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nexport async function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,sBACvC,GAAG;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;AAEA,eAAsB,wBACpB,IACA,IACA,mBAIA,OACA,WACA,WACA,eAAkC,YAClC,qBAAqB,MACrB;CACA,MAAM,EAAC,cAAc,gBAAgB,cAAa;CAClD,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,QAA+B,CAAC;EACtC,IAAI,gBAAgB;EACpB,MAAM,UAAU,MAAM,GAMZ;;aAED,IAAI,MAAM,EAAE;EAErB,IAAI,QAAQ,QAAQ;GAClB,MAAM,EAAC,gBAAgB,iBAAgB,QAAQ;GAC/C,IACE,mBAAmB,cAAc,kBACjC,CAAC,OAAO,IAAI,IAAI,YAAY,GAAG,IAAI,IAAI,cAAc,YAAY,CAAC,GAClE;IACA,IAAI,cAAc,mBAAmB,WACnC,MAAM,IAAI,gBACR,0BAA0B,eAAe,sBACpB,cAAc,eAAe,kBAC9B,WACtB;IAEF,GAAG,OACD,uBAAuB,eAAe,iCACnB,cAAc,eAAe,mBAClD;IAGA,WAAgB,QAAQ;IAWxB,gBAAgB;IAChB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAI,qBACA,CAAC,GAAG,kBAAkB,IAAI,MAAM,EAAE,aAAa,IAC/C,CAAC,GACL,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;GACF;EACF;EAEA,IAAI,QAAQ,WAAW,KAAK,eAAe;GAQzC,MAAM,YAAY,cAAc;GAChC,MAAM,YAAkC,qBACpC,CACE;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,QAAO;GAAC,GAC1C;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,SAAQ;GAAC,CAC7C,IACA,CAAC;GAEL,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;iDAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;GAEA,IAAI,eAAe;IAKjB,MAAM,QAAQ,aAAa,YAAY;KACrC,GAAG,OACD,2DACF;KACA,MAAM,6BAA6B,IAAI,IAAI,KAAK;IAClD,GAAG,gCAAgC;IACnC,IAAI;KACF,OAAO,MAAM,QAAQ,IAAI,KAAK;IAChC,UAAU;KACR,aAAa,KAAK;IACpB;GACF;GACA,OAAO,QAAQ,IAAI,KAAK;EAC1B;EAEA,MAAM,EAAC,kBAAiB,QAAQ;EAChC,IAAI,eAAe;GACjB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;GAElE,GAAG,QACD,oOAKF;EACF;EAEA,OAAO,CAAC;CACV,CAAC;AACH;AAIA,IAAM,mCAAmC;AAEzC,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}

package/out/zero-cache/src/services/change-streamer/snapshot.d.ts

@@ -14,6 +14,7 @@
 import type { LogContext } from '@rocicorp/logger';
 import * as v from '../../../../shared/src/valita.ts';
 import { type NormalizedZeroConfig } from '../../config/normalize.ts';
+import type { Source } from '../../types/streams.ts';
 declare const statusSchema: v.ObjectType<{
     tag: v.Type<"status">;
     /**
@@ -79,6 +80,7 @@ export declare const snapshotMessageSchema: v.UnionType<[v.TupleType<[v.Type<"st
     minWatermark: v.Type<string>;
 }, undefined>]>]>;
 export type SnapshotMessage = v.Infer<typeof statusMessageSchema>;
-export declare function reserveAndGetSnapshotStatus(lc: LogContext, config: NormalizedZeroConfig): Promise<SnapshotStatus>;
+export type ReserveSnapshot = (lc: LogContext, config: NormalizedZeroConfig) => Promise<Source<SnapshotMessage>>;
+export declare function reserveAndGetSnapshotStatus(lc: LogContext, config: NormalizedZeroConfig, reserve?: ReserveSnapshot): Promise<SnapshotStatus>;
 export {};
 //# sourceMappingURL=snapshot.d.ts.map

package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,EAAC,KAAK,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAKpE,QAAA,MAAM,YAAY;;IAGhB;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;aAEH,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAE1D,QAAA,MAAM,mBAAmB;;IAxBvB;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;eAMmE,CAAC;AAEzE,eAAO,MAAM,qBAAqB;;IA1BhC;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;iBAQ4D,CAAC;AAElE,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAElE,wBAAgB,2BAA2B,CACzC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAiDzB"}
+{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,EAAC,KAAK,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,QAAA,MAAM,YAAY;;IAGhB;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;aAEH,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAE1D,QAAA,MAAM,mBAAmB;;IAxBvB;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;eAMmE,CAAC;AAEzE,eAAO,MAAM,qBAAqB;;IA1BhC;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;iBAQ4D,CAAC;AAElE,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAElE,MAAM,MAAM,eAAe,GAAG,CAC5B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,KACzB,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;AAEtC,wBAAgB,2BAA2B,CACzC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,OAAO,GAAE,eAAiC,GACzC,OAAO,CAAC,cAAc,CAAC,CAwEzB"}

package/out/zero-cache/src/services/change-streamer/snapshot.js

@@ -1,3 +1,4 @@
+import { AbortError } from "../../../../shared/src/abort-error.js";
 import { sleep } from "../../../../shared/src/sleep.js";
 import { valita_exports } from "../../../../shared/src/valita.js";
 import { getShardConfig } from "../../types/shards.js";
@@ -28,35 +29,68 @@ var statusSchema = valita_exports.object({
 });
 var statusMessageSchema = valita_exports.tuple([valita_exports.literal("status"), statusSchema]);
 var snapshotMessageSchema = valita_exports.union(statusMessageSchema);
-function reserveAndGetSnapshotStatus(lc, config) {
+function reserveAndGetSnapshotStatus(lc, config, reserve = reserveSnapshot) {
 	const { promise: status, resolve, reject } = resolver();
 	(async function() {
 		const abort = new AbortController();
-		process.on("SIGINT", () => abort.abort());
-		process.on("SIGTERM", () => abort.abort());
-		for (let i = 0;; i++) {
-			let err;
-			try {
-				let resolved = false;
-				const stream = await reserveSnapshot(lc, config);
-				for await (const msg of stream) {
-					resolve(msg[1]);
-					resolved = true;
+		const { signal } = abort;
+		const onSignal = () => abort.abort();
+		process.on("SIGINT", onSignal);
+		process.on("SIGTERM", onSignal);
+		try {
+			for (let i = 0;; i++) {
+				let err;
+				try {
+					let resolved = false;
+					const stream = await untilAborted(reserve(lc, config), signal);
+					const cancelStream = () => stream.cancel(new AbortError("Aborted"));
+					signal.addEventListener("abort", cancelStream, { once: true });
+					try {
+						for await (const msg of stream) {
+							resolve(msg[1]);
+							resolved = true;
+						}
+					} finally {
+						signal.removeEventListener("abort", cancelStream);
+					}
+					if (resolved) break;
+				} catch (e) {
+					err = e;
+				}
+				if (signal.aborted) return reject(err instanceof AbortError ? err : new AbortError("Aborted"));
+				lc.warn?.(`Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`, String(err));
+				try {
+					await sleep(5e3, abort.signal);
+				} catch (e) {
+					return reject(e);
 				}
-				if (resolved) break;
-			} catch (e) {
-				err = e;
-			}
-			lc.warn?.(`Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`, String(err));
-			try {
-				await sleep(5e3, abort.signal);
-			} catch (e) {
-				return reject(e);
 			}
+		} finally {
+			process.off("SIGINT", onSignal);
+			process.off("SIGTERM", onSignal);
 		}
 	})();
 	return status;
 }
+/**
+* Resolves with the reserved stream, or rejects with an AbortError if the
+* `signal` is aborted while the reservation is pending (in which case a
+* stream that arrives later is canceled rather than held open).
+*/
+function untilAborted(reservation, signal) {
+	return new Promise((resolve, reject) => {
+		const onAbort = () => reject(new AbortError("Aborted"));
+		signal.addEventListener("abort", onAbort, { once: true });
+		reservation.then((stream) => {
+			signal.removeEventListener("abort", onAbort);
+			if (signal.aborted) stream.cancel();
+			else resolve(stream);
+		}, (e) => {
+			signal.removeEventListener("abort", onAbort);
+			reject(e);
+		});
+	});
+}
 function reserveSnapshot(lc, config) {
 	const { taskID, change, changeStreamer } = config;
 	return new ChangeStreamerHttpClient(lc, getShardConfig(config), change.db, changeStreamer.uri).reserveSnapshot(taskID);

package/out/zero-cache/src/services/change-streamer/snapshot.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"sourcesContent":["/**\n * The `snapshot` API serves the purpose of:\n * - informing subscribers (i.e. view-syncers) of the (litestream)\n *   backup location from which to restore a replica snapshot\n * - checking whether a restored backup or existing replica is\n *   compatible with the change-streamer\n * - preventing change-log cleanup while a snapshot restore is in\n *   progress\n * - tracking the approximate time it takes from the beginning of\n *   snapshot \"reservation\" to the subsequent subscription, which\n *   serves as the minimum interval to wait before cleaning up\n *   backed up changes.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {type NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from './change-streamer-http.ts';\n\nconst statusSchema = v.object({\n  tag: v.literal('status'),\n\n  /**\n   * The location from which litestream should perform the restore.\n   */\n  backupURL: v.string(),\n\n  /**\n   * The `replicaVersion` of the backup. If a subscriber's restored or\n   * existing replica is of a different version, it should delete it and\n   * retry the restore from litestream (i.e. equivalent to a\n   * `WrongReplicaVersion` response from a `/changes` subscription).\n   */\n  replicaVersion: v.string(),\n\n  /**\n   * The earliest watermark from which catchup is possible. If the\n   * subscriber's replica is older that this watermark, it should delete it\n   * and (retry the) restore from litestream (i.e. equivalent to a\n   * `WatermarkTooOld` response from a `/changes` subscription).\n   */\n  minWatermark: v.string(),\n});\n\nexport type SnapshotStatus = v.Infer<typeof statusSchema>;\n\nconst statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\nexport const snapshotMessageSchema = v.union(statusMessageSchema);\n\nexport type SnapshotMessage = v.Infer<typeof statusMessageSchema>;\n\nexport function reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // backup monitor returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;AAuBA,IAAM,eAAe,eAAE,OAAO;CAC5B,KAAK,eAAE,QAAQ,QAAQ;;;;CAKvB,WAAW,eAAE,OAAO;;;;;;;CAQpB,gBAAgB,eAAE,OAAO;;;;;;;CAQzB,cAAc,eAAE,OAAO;AACzB,CAAC;AAID,IAAM,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAEvE,IAAa,wBAAwB,eAAE,MAAM,mBAAmB;AAIhE,SAAgB,4BACd,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}
+{"version":3,"file":"snapshot.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"sourcesContent":["/**\n * The `snapshot` API serves the purpose of:\n * - informing subscribers (i.e. view-syncers) of the (litestream)\n *   backup location from which to restore a replica snapshot\n * - checking whether a restored backup or existing replica is\n *   compatible with the change-streamer\n * - preventing change-log cleanup while a snapshot restore is in\n *   progress\n * - tracking the approximate time it takes from the beginning of\n *   snapshot \"reservation\" to the subsequent subscription, which\n *   serves as the minimum interval to wait before cleaning up\n *   backed up changes.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {type NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from './change-streamer-http.ts';\n\nconst statusSchema = v.object({\n  tag: v.literal('status'),\n\n  /**\n   * The location from which litestream should perform the restore.\n   */\n  backupURL: v.string(),\n\n  /**\n   * The `replicaVersion` of the backup. If a subscriber's restored or\n   * existing replica is of a different version, it should delete it and\n   * retry the restore from litestream (i.e. equivalent to a\n   * `WrongReplicaVersion` response from a `/changes` subscription).\n   */\n  replicaVersion: v.string(),\n\n  /**\n   * The earliest watermark from which catchup is possible. If the\n   * subscriber's replica is older that this watermark, it should delete it\n   * and (retry the) restore from litestream (i.e. equivalent to a\n   * `WatermarkTooOld` response from a `/changes` subscription).\n   */\n  minWatermark: v.string(),\n});\n\nexport type SnapshotStatus = v.Infer<typeof statusSchema>;\n\nconst statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\nexport const snapshotMessageSchema = v.union(statusMessageSchema);\n\nexport type SnapshotMessage = v.Infer<typeof statusMessageSchema>;\n\nexport type ReserveSnapshot = (\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n) => Promise<Source<SnapshotMessage>>;\n\nexport function reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  reserve: ReserveSnapshot = reserveSnapshot, // for testing\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    const {signal} = abort;\n    const onSignal = () => abort.abort();\n    process.on('SIGINT', onSignal);\n    process.on('SIGTERM', onSignal);\n\n    try {\n      for (let i = 0; ; i++) {\n        let err: unknown;\n        try {\n          let resolved = false;\n          const stream = await untilAborted(reserve(lc, config), signal);\n          // A signal while the stream is open cancels it, which ends the\n          // iteration below with an AbortError.\n          const cancelStream = () => stream.cancel(new AbortError('Aborted'));\n          signal.addEventListener('abort', cancelStream, {once: true});\n          try {\n            for await (const msg of stream) {\n              // Capture the value of the status message that the change-streamer\n              // backup monitor returns, and hold the connection open to\n              // \"reserve\" the snapshot and prevent change log cleanup.\n              resolve(msg[1]);\n              resolved = true;\n            }\n          } finally {\n            signal.removeEventListener('abort', cancelStream);\n          }\n          // The change-streamer itself closes the connection when the\n          // subscription is started (or the reservation retried).\n          if (resolved) {\n            break;\n          }\n        } catch (e) {\n          err = e;\n        }\n        if (signal.aborted) {\n          // (A no-op if the status was already resolved.)\n          return reject(\n            err instanceof AbortError ? err : new AbortError('Aborted'),\n          );\n        }\n        // Retry in the view-syncer since it cannot proceed until it connects\n        // to a (compatible) replication-manager. In particular, a\n        // replication-manager that does not support the view-syncer's\n        // change-streamer protocol will close the stream with an error; this\n        // retry logic essentially delays the startup of a view-syncer until\n        // a compatible replication-manager has been rolled out, allowing\n        // replication-manager and view-syncer services to be updated in\n        // parallel.\n        lc.warn?.(\n          `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n          String(err),\n        );\n        try {\n          await sleep(5000, abort.signal);\n        } catch (e) {\n          return reject(e);\n        }\n      }\n    } finally {\n      // This function is called repeatedly (e.g. by the replicator's restore\n      // loop), so the signal handlers must not outlive the reservation.\n      process.off('SIGINT', onSignal);\n      process.off('SIGTERM', onSignal);\n    }\n  })();\n\n  return status;\n}\n\n/**\n * Resolves with the reserved stream, or rejects with an AbortError if the\n * `signal` is aborted while the reservation is pending (in which case a\n * stream that arrives later is canceled rather than held open).\n */\nfunction untilAborted<T>(\n  reservation: Promise<Source<T>>,\n  signal: AbortSignal,\n): Promise<Source<T>> {\n  return new Promise((resolve, reject) => {\n    const onAbort = () => reject(new AbortError('Aborted'));\n    signal.addEventListener('abort', onAbort, {once: true});\n    reservation.then(\n      stream => {\n        signal.removeEventListener('abort', onAbort);\n        if (signal.aborted) {\n          stream.cancel();\n        } else {\n          resolve(stream);\n        }\n      },\n      e => {\n        signal.removeEventListener('abort', onAbort);\n        reject(e);\n      },\n    );\n  });\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;AAwBA,IAAM,eAAe,eAAE,OAAO;CAC5B,KAAK,eAAE,QAAQ,QAAQ;;;;CAKvB,WAAW,eAAE,OAAO;;;;;;;CAQpB,gBAAgB,eAAE,OAAO;;;;;;;CAQzB,cAAc,eAAE,OAAO;AACzB,CAAC;AAID,IAAM,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAEvE,IAAa,wBAAwB,eAAE,MAAM,mBAAmB;AAShE,SAAgB,4BACd,IACA,QACA,UAA2B,iBACF;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,MAAM,EAAC,WAAU;EACjB,MAAM,iBAAiB,MAAM,MAAM;EACnC,QAAQ,GAAG,UAAU,QAAQ;EAC7B,QAAQ,GAAG,WAAW,QAAQ;EAE9B,IAAI;GACF,KAAK,IAAI,IAAI,IAAK,KAAK;IACrB,IAAI;IACJ,IAAI;KACF,IAAI,WAAW;KACf,MAAM,SAAS,MAAM,aAAa,QAAQ,IAAI,MAAM,GAAG,MAAM;KAG7D,MAAM,qBAAqB,OAAO,OAAO,IAAI,WAAW,SAAS,CAAC;KAClE,OAAO,iBAAiB,SAAS,cAAc,EAAC,MAAM,KAAI,CAAC;KAC3D,IAAI;MACF,WAAW,MAAM,OAAO,QAAQ;OAI9B,QAAQ,IAAI,EAAE;OACd,WAAW;MACb;KACF,UAAU;MACR,OAAO,oBAAoB,SAAS,YAAY;KAClD;KAGA,IAAI,UACF;IAEJ,SAAS,GAAG;KACV,MAAM;IACR;IACA,IAAI,OAAO,SAET,OAAO,OACL,eAAe,aAAa,MAAM,IAAI,WAAW,SAAS,CAC5D;IAUF,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;IACA,IAAI;KACF,MAAM,MAAM,KAAM,MAAM,MAAM;IAChC,SAAS,GAAG;KACV,OAAO,OAAO,CAAC;IACjB;GACF;EACF,UAAU;GAGR,QAAQ,IAAI,UAAU,QAAQ;GAC9B,QAAQ,IAAI,WAAW,QAAQ;EACjC;CACF,GAAG;CAEH,OAAO;AACT;;;;;;AAOA,SAAS,aACP,aACA,QACoB;CACpB,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,MAAM,gBAAgB,OAAO,IAAI,WAAW,SAAS,CAAC;EACtD,OAAO,iBAAiB,SAAS,SAAS,EAAC,MAAM,KAAI,CAAC;EACtD,YAAY,MACV,WAAU;GACR,OAAO,oBAAoB,SAAS,OAAO;GAC3C,IAAI,OAAO,SACT,OAAO,OAAO;QAEd,QAAQ,MAAM;EAElB,IACA,MAAK;GACH,OAAO,oBAAoB,SAAS,OAAO;GAC3C,OAAO,CAAC;EACV,CACF;CACF,CAAC;AACH;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts

@@ -25,6 +25,11 @@ export type SQLiteChangeLogWriterOptions = {
     onRebuilt?: (() => void) | undefined;
     /** Overridable for tests. */
     now?: (() => number) | undefined;
+    /**
+     * Percentage of transactions that retain the rollout-only per-message
+     * timing and source-vs-stored hash validation. Defaults to 1.
+     */
+    shadowValidationPercent?: number | undefined;
 };
 /**
  * Appends the canonical change stream to the SQLite change log, from the
@@ -98,7 +103,7 @@ export declare class SQLiteChangeLogWriter {
      * Writes one stream message, committing at transaction boundaries. Never
      * throws: a write failure disables the writer instead of failing the stream.
      */
-    write(change: ChangeStreamData, json: string): void;
+    write(change: ChangeStreamData, json: string, storedChange?: string): void;
     /**
      * Discards the open transaction, if any. Called when the change stream is
      * interrupted mid-transaction.

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-writer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAM3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AAKtF,OAAO,EAUL,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EAE1B,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAML,KAAK,iCAAiC,EACvC,MAAM,kDAAkD,CAAC;AAE1D,MAAM,MAAM,4BAA4B,GAAG;IACzC,iFAAiF;IACjF,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,QAAQ,EAAE,iBAAiB,CAAC;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IACrD;;;;;OAKG;IACH,UAAU,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC;IACtC,6EAA6E;IAC7E,SAAS,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC;IACrC,6BAA6B;IAC7B,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,qBAAqB;;IAUhC,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,4BAA4B,EAI7D;IAED,qEAAqE;IACrE,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;;;;;OAOG;IACH,IAAI,UAAU,IAAI,QAAQ,GAAG,SAAS,CAErC;IAED,2EAA2E;IAC3E,KAAK,IAAI,iCAAiC,GAAG,SAAS,CAErD;IAED;;;;;;;;;;;;;OAaG;IACH,SAAS,CACP,UAAU,EAAE,oBAAoB,GAC/B,oBAAoB,GAAG,SAAS,CAElC;IAED;;;;;OAKG;IACH,gBAAgB,CACd,IAAI,EAAE,MAAM,oBAAoB,GAC/B,oBAAoB,GAAG,SAAS,CAElC;IA2GD;;;OAGG;IACH,KAAK,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAgElD;IAED;;;OAGG;IACH,KAAK,IAAI,IAAI,CAWZ;IAED,KAAK,IAAI,IAAI,CAWZ;CA6BF"}
+{"version":3,"file":"sqlite-change-log-writer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAM3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AAKtF,OAAO,EAUL,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EAE1B,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAML,KAAK,iCAAiC,EACvC,MAAM,kDAAkD,CAAC;AAG1D,MAAM,MAAM,4BAA4B,GAAG;IACzC,iFAAiF;IACjF,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,QAAQ,EAAE,iBAAiB,CAAC;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IACrD;;;;;OAKG;IACH,UAAU,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC;IACtC,6EAA6E;IAC7E,SAAS,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC;IACrC,6BAA6B;IAC7B,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAIF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,qBAAqB;;IAYhC,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,4BAA4B,EAY7D;IAED,qEAAqE;IACrE,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;;;;;OAOG;IACH,IAAI,UAAU,IAAI,QAAQ,GAAG,SAAS,CAErC;IAED,2EAA2E;IAC3E,KAAK,IAAI,iCAAiC,GAAG,SAAS,CAErD;IAED;;;;;;;;;;;;;OAaG;IACH,SAAS,CACP,UAAU,EAAE,oBAAoB,GAC/B,oBAAoB,GAAG,SAAS,CAElC;IAED;;;;;OAKG;IACH,gBAAgB,CACd,IAAI,EAAE,MAAM,oBAAoB,GAC/B,oBAAoB,GAAG,SAAS,CAElC;IA2GD;;;OAGG;IACH,KAAK,CACH,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,MAAM,EACZ,YAAY,SAA8C,GACzD,IAAI,CA4FN;IAeD;;;OAGG;IACH,KAAK,IAAI,IAAI,CAYZ;IAED,KAAK,IAAI,IAAI,CAYZ;CA6BF"}