@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.20

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts

@@ -33,9 +33,9 @@ export declare class Forwarder {
      * two components have an equivalent interpretation of whether a Transaction is
      * currently being streamed.
      *
-     * This version of forward is fire-and-forget, with no flow control. The
-     * change-streamer should call and await {@link forwardWithFlowControl()}
-     * occasionally to avoid memory blowup.
+     * This version of forward buffers changes across the current event loop turn
+     * and dispatches them in batches. The change-streamer should call and await
+     * {@link forwardWithFlowControl()} occasionally to avoid memory blowup.
      */
     forward(entry: WatermarkedChange): void;
     sendStatus(status: Status): void;

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,MAAM,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE/E,OAAO,KAAK,EAAC,UAAU,EAAkB,MAAM,iBAAiB,CAAC;AAEjE,MAAM,MAAM,sBAAsB,GAAG;IACnC,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5D,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,qBAAa,SAAS;;IA+BpB,YACE,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAEL,EA6DF;IAED,oBAAoB,SAMnB;IAED;;;;;;;OAOG;IACH,uBAAuB,CAAC,GAAG,EAAE,MAAM,QA6DlC;IAED,mBAAmB,SAElB;IAED;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU,QAMlB;IAED,MAAM,CAAC,GAAG,EAAE,UAAU,QAIrB;IAED;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB,QAG/B;IAED,UAAU,CAAC,MAAM,EAAE,MAAM,QAIxB;IAED;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB,iBAyBpD;IA6CD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,CAOrB;CACF"}
+{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,MAAM,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE/E,OAAO,KAAK,EAAC,UAAU,EAAkB,MAAM,iBAAiB,CAAC;AAEjE,MAAM,MAAM,sBAAsB,GAAG;IACnC,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5D,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAIF,qBAAa,SAAS;;IAmCpB,YACE,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAEL,EA6DF;IAED,oBAAoB,SAMnB;IAED;;;;;;;OAOG;IACH,uBAAuB,CAAC,GAAG,EAAE,MAAM,QA6DlC;IAED,mBAAmB,SAGlB;IAED;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU,QAOlB;IAED,MAAM,CAAC,GAAG,EAAE,UAAU,QAKrB;IAED;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB,QAmB/B;IAED,UAAU,CAAC,MAAM,EAAE,MAAM,QAKxB;IAED;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB,iBA4BpD;IAkED,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,CAOrB;CACF"}

package/out/zero-cache/src/services/change-streamer/forwarder.js

@@ -2,6 +2,7 @@ import { joinIterables, wrapIterable } from "../../../../shared/src/iterables.js
 import { getOrCreateCounter, getOrCreateGauge, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
 import { Broadcast } from "./broadcast.js";
 //#region ../zero-cache/src/services/change-streamer/forwarder.ts
+var FORWARD_BATCH_SIZE = 64;
 var Forwarder = class {
 	#lc;
 	#setTimeout;
@@ -13,6 +14,9 @@ var Forwarder = class {
 	#flowControlWaits = getOrCreateCounter("replication", "flow_control.waits", "Flow-control checkpoints completed, labeled by release mode.");
 	#flowControlWaitTime = getOrCreateLatencyHistogram("replication", "flow_control.wait_duration", "Time replication waits at flow-control checkpoints.");
 	#inTransaction = false;
+	#pending = [];
+	#flushScheduled = false;
+	#flushImmediateId = void 0;
 	#currentBroadcast;
 	#progressMonitor;
 	#backupLagEventCount = 0;
@@ -79,6 +83,7 @@ var Forwarder = class {
 	}
 	stopProgressMonitor() {
 		clearInterval(this.#progressMonitor);
+		this.#flushPendingWithoutTracking();
 	}
 	/**
 	* `add()` is called in lock step with `Storer.catchup()` so that the
@@ -86,10 +91,12 @@ var Forwarder = class {
 	* currently being streamed.
 	*/
 	add(sub) {
+		this.#flushPendingWithoutTracking();
 		if (this.#inTransaction) this.#queued.add(sub);
 		else this.#active.add(sub);
 	}
 	remove(sub) {
+		this.#flushPendingWithoutTracking();
 		this.#active.delete(sub);
 		this.#queued.delete(sub);
 		sub.close();
@@ -99,15 +106,22 @@ var Forwarder = class {
 	* two components have an equivalent interpretation of whether a Transaction is
 	* currently being streamed.
 	*
-	* This version of forward is fire-and-forget, with no flow control. The
-	* change-streamer should call and await {@link forwardWithFlowControl()}
-	* occasionally to avoid memory blowup.
+	* This version of forward buffers changes across the current event loop turn
+	* and dispatches them in batches. The change-streamer should call and await
+	* {@link forwardWithFlowControl()} occasionally to avoid memory blowup.
 	*/
 	forward(entry) {
-		Broadcast.withoutTracking(this.#active.values(), entry);
+		this.#pending.push(entry);
+		if (this.#queued.size > 0 && (entry[1] === "commit" || entry[1] === "rollback")) this.#flushPendingWithoutTracking();
 		this.#updateActiveSubscribers(entry[1]);
+		if (this.#pending.length >= FORWARD_BATCH_SIZE) this.#flushPendingWithoutTracking();
+		else if (!this.#flushScheduled && this.#pending.length > 0) {
+			this.#flushScheduled = true;
+			this.#flushImmediateId = setImmediate(() => this.#flushPendingWithoutTracking());
+		}
 	}
 	sendStatus(status) {
+		this.#flushPendingWithoutTracking();
 		for (const sub of this.#active.values()) sub.sendStatus(status);
 	}
 	/**
@@ -115,8 +129,10 @@ var Forwarder = class {
 	* Promise that resolves when replication should continue.
 	*/
 	async forwardWithFlowControl(entry) {
+		this.#pending.push(entry);
+		const batch = this.#drainPending();
 		const start = performance.now();
-		const broadcast = new Broadcast(this.#lc, this.#active.values(), entry, this.#earlyReleaseOptions);
+		const broadcast = new Broadcast(this.#lc, this.#active.values(), batch, this.#earlyReleaseOptions);
 		this.#updateActiveSubscribers(entry[1]);
 		this.#currentBroadcast = broadcast;
 		try {
@@ -128,6 +144,21 @@ var Forwarder = class {
 			if (this.#currentBroadcast === broadcast) this.#currentBroadcast = void 0;
 		}
 	}
+	#drainPending() {
+		if (this.#flushImmediateId !== void 0) {
+			clearImmediate(this.#flushImmediateId);
+			this.#flushImmediateId = void 0;
+		}
+		this.#flushScheduled = false;
+		if (this.#pending.length === 0) return [];
+		const batch = this.#pending;
+		this.#pending = [];
+		return batch;
+	}
+	#flushPendingWithoutTracking() {
+		const batch = this.#drainPending();
+		if (batch.length > 0) Broadcast.withoutTracking(this.#active.values(), batch);
+	}
 	#subscriberStats() {
 		let pendingMessages = 0;
 		let backlogMessages = 0;

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#backupLagEventCount","#lastBackupLagWarnMs","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #earlyReleaseOptions: EarlyReleaseOptions | undefined;\n  readonly #flowControlSlowSubscriberGracePeriodMs: number | undefined;\n\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  // The backup-replicator is never disconnected as a laggard, so its lag\n  // detections are logged (rather than acted upon). Track the running total and\n  // throttle the warning to at most once per grace period to avoid spamming the\n  // log for the duration of a slow catchup.\n  #backupLagEventCount = 0;\n  #lastBackupLagWarnMs: number | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {\n      flowControlConsensusTimeoutProportion: 2.0,\n    },\n  ) {\n    this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    this.#earlyReleaseOptions =\n      opts.flowControlConsensusTimeoutProportion < 0\n        ? undefined\n        : {\n            consensusTimeoutProportion:\n              opts.flowControlConsensusTimeoutProportion,\n            setTimeoutFn: this.#setTimeout,\n            clearTimeoutFn: this.#clearTimeout,\n          };\n    this.#flowControlSlowSubscriberGracePeriodMs =\n      opts.flowControlSlowSubscriberGracePeriodMs;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(\n      () => this.checkSubscriberProgress(performance.now()),\n      1000,\n    );\n  }\n\n  /**\n   * Runs one progress-monitor tick: samples each active subscriber's change\n   * rate, logs slow broadcasts, and disconnects subscribers that have been\n   * continuously lagging for longer than the configured grace period.\n   *\n   * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n   * than private) so tests can drive a single tick with an explicit `now`.\n   */\n  checkSubscriberProgress(now: number) {\n    let timedOut: Map<Subscriber, SubscriberStats> | undefined;\n    let slowestOnTime: SubscriberStats | undefined;\n\n    for (const sub of this.#active) {\n      const stats = sub.sampleProcessRate(now).getStats();\n      if (stats.missedLastTimeout) {\n        (timedOut ??= new Map()).set(sub, stats);\n      } else if (\n        stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n      ) {\n        slowestOnTime = stats;\n      }\n    }\n    this.#currentBroadcast?.logSlowRequests(now);\n\n    if (\n      this.#flowControlSlowSubscriberGracePeriodMs &&\n      timedOut?.size &&\n      slowestOnTime\n    ) {\n      for (const [sub, stats] of timedOut.entries()) {\n        const rate =\n          stats.processRate < slowestOnTime.processRate\n            ? 'lagging'\n            : 'catching-up';\n        const laggingDuration = sub.reportChangeRate(now, rate);\n        if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n          if (sub.mode === 'backup') {\n            // The backup-replicator can only be detected as a laggard during\n            // initial catchup; after that it is required for consensus. If it\n            // triggers laggard detection in the former case, just log and\n            // continue; as disconnecting it would otherwise kill the\n            // change-streamer task altogether.\n            this.#backupLagEventCount++;\n            const grace = this.#flowControlSlowSubscriberGracePeriodMs;\n            if (\n              this.#lastBackupLagWarnMs === undefined ||\n              now - this.#lastBackupLagWarnMs >= grace\n            ) {\n              this.#lastBackupLagWarnMs = now;\n              this.#lc.warn?.(\n                `backup subscriber ${sub.id} has lagged for ${laggingDuration}ms ` +\n                  `(${this.#backupLagEventCount} lag events so far). ` +\n                  `this is only expected during initial catchup`,\n                {stats},\n              );\n            }\n          } else {\n            this.#lc.warn?.(\n              `serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n              {stats},\n            );\n            sub.close(\n              StreamTooFarBehind,\n              `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n            );\n          }\n        }\n      }\n    }\n  }\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const start = performance.now();\n    const broadcast = new Broadcast(\n      this.#lc,\n      this.#active.values(),\n      entry,\n      this.#earlyReleaseOptions,\n    );\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAoBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAMA,uBAAuB;CACvB;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAC1B,IAAI,IAAI,SAAS,UAAU;IAMzB,KAAKM;IACL,MAAM,QAAQ,KAAKN;IACnB,IACE,KAAKO,yBAAyB,KAAA,KAC9B,MAAM,KAAKA,wBAAwB,OACnC;KACA,KAAKA,uBAAuB;KAC5B,KAAKb,IAAI,OACP,qBAAqB,IAAI,GAAG,kBAAkB,gBAAgB,MACxD,KAAKY,qBAAqB,oEAEhC,EAAC,MAAK,CACR;IACF;GACF,OAAO;IACL,KAAKZ,IAAI,OACP,sBAAsB,IAAI,GAAG,kBAAkB,gBAAgB,oBAC/D,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EAEJ;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKI,gBACP,KAAKV,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKY,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKZ,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKH,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKU,yBAAyB,MAAM,EAAE;EAGtC,KAAKJ,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKR,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKU,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKV,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}
+{"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#backupLagEventCount","#lastBackupLagWarnMs","#flushPendingWithoutTracking","#inTransaction","#pending","#updateActiveSubscribers","#flushScheduled","#flushImmediateId","#drainPending"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nconst FORWARD_BATCH_SIZE = 64;\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #earlyReleaseOptions: EarlyReleaseOptions | undefined;\n  readonly #flowControlSlowSubscriberGracePeriodMs: number | undefined;\n\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #pending: WatermarkedChange[] = [];\n  #flushScheduled = false;\n  #flushImmediateId: NodeJS.Immediate | undefined = undefined;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  // The backup-replicator is never disconnected as a laggard, so its lag\n  // detections are logged (rather than acted upon). Track the running total and\n  // throttle the warning to at most once per grace period to avoid spamming the\n  // log for the duration of a slow catchup.\n  #backupLagEventCount = 0;\n  #lastBackupLagWarnMs: number | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {\n      flowControlConsensusTimeoutProportion: 2.0,\n    },\n  ) {\n    this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    this.#earlyReleaseOptions =\n      opts.flowControlConsensusTimeoutProportion < 0\n        ? undefined\n        : {\n            consensusTimeoutProportion:\n              opts.flowControlConsensusTimeoutProportion,\n            setTimeoutFn: this.#setTimeout,\n            clearTimeoutFn: this.#clearTimeout,\n          };\n    this.#flowControlSlowSubscriberGracePeriodMs =\n      opts.flowControlSlowSubscriberGracePeriodMs;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(\n      () => this.checkSubscriberProgress(performance.now()),\n      1000,\n    );\n  }\n\n  /**\n   * Runs one progress-monitor tick: samples each active subscriber's change\n   * rate, logs slow broadcasts, and disconnects subscribers that have been\n   * continuously lagging for longer than the configured grace period.\n   *\n   * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n   * than private) so tests can drive a single tick with an explicit `now`.\n   */\n  checkSubscriberProgress(now: number) {\n    let timedOut: Map<Subscriber, SubscriberStats> | undefined;\n    let slowestOnTime: SubscriberStats | undefined;\n\n    for (const sub of this.#active) {\n      const stats = sub.sampleProcessRate(now).getStats();\n      if (stats.missedLastTimeout) {\n        (timedOut ??= new Map()).set(sub, stats);\n      } else if (\n        stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n      ) {\n        slowestOnTime = stats;\n      }\n    }\n    this.#currentBroadcast?.logSlowRequests(now);\n\n    if (\n      this.#flowControlSlowSubscriberGracePeriodMs &&\n      timedOut?.size &&\n      slowestOnTime\n    ) {\n      for (const [sub, stats] of timedOut.entries()) {\n        const rate =\n          stats.processRate < slowestOnTime.processRate\n            ? 'lagging'\n            : 'catching-up';\n        const laggingDuration = sub.reportChangeRate(now, rate);\n        if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n          if (sub.mode === 'backup') {\n            // The backup-replicator can only be detected as a laggard during\n            // initial catchup; after that it is required for consensus. If it\n            // triggers laggard detection in the former case, just log and\n            // continue; as disconnecting it would otherwise kill the\n            // change-streamer task altogether.\n            this.#backupLagEventCount++;\n            const grace = this.#flowControlSlowSubscriberGracePeriodMs;\n            if (\n              this.#lastBackupLagWarnMs === undefined ||\n              now - this.#lastBackupLagWarnMs >= grace\n            ) {\n              this.#lastBackupLagWarnMs = now;\n              this.#lc.warn?.(\n                `backup subscriber ${sub.id} has lagged for ${laggingDuration}ms ` +\n                  `(${this.#backupLagEventCount} lag events so far). ` +\n                  `this is only expected during initial catchup`,\n                {stats},\n              );\n            }\n          } else {\n            this.#lc.warn?.(\n              `serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n              {stats},\n            );\n            sub.close(\n              StreamTooFarBehind,\n              `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n            );\n          }\n        }\n      }\n    }\n  }\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#flushPendingWithoutTracking();\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    this.#flushPendingWithoutTracking();\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#flushPendingWithoutTracking();\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward buffers changes across the current event loop turn\n   * and dispatches them in batches. The change-streamer should call and await\n   * {@link forwardWithFlowControl()} occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    this.#pending.push(entry);\n\n    if (\n      this.#queued.size > 0 &&\n      (entry[1] === 'commit' || entry[1] === 'rollback')\n    ) {\n      this.#flushPendingWithoutTracking();\n    }\n    this.#updateActiveSubscribers(entry[1]);\n\n    if (this.#pending.length >= FORWARD_BATCH_SIZE) {\n      this.#flushPendingWithoutTracking();\n    } else if (!this.#flushScheduled && this.#pending.length > 0) {\n      this.#flushScheduled = true;\n      this.#flushImmediateId = setImmediate(() =>\n        this.#flushPendingWithoutTracking(),\n      );\n    }\n  }\n\n  sendStatus(status: Status) {\n    this.#flushPendingWithoutTracking();\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    this.#pending.push(entry);\n\n    const batch = this.#drainPending();\n    const start = performance.now();\n    const broadcast = new Broadcast(\n      this.#lc,\n      this.#active.values(),\n      batch,\n      this.#earlyReleaseOptions,\n    );\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #drainPending(): WatermarkedChange[] {\n    if (this.#flushImmediateId !== undefined) {\n      clearImmediate(this.#flushImmediateId);\n      this.#flushImmediateId = undefined;\n    }\n    this.#flushScheduled = false;\n    if (this.#pending.length === 0) {\n      return [];\n    }\n    const batch = this.#pending;\n    this.#pending = [];\n    return batch;\n  }\n\n  #flushPendingWithoutTracking() {\n    const batch = this.#drainPending();\n    if (batch.length > 0) {\n      Broadcast.withoutTracking(this.#active.values(), batch);\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAoBA,IAAM,qBAAqB;AAE3B,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB,WAAgC,CAAC;CACjC,kBAAkB;CAClB,oBAAkD,KAAA;CAElD;CACA;CAMA,uBAAuB;CACvB;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAC1B,IAAI,IAAI,SAAS,UAAU;IAMzB,KAAKM;IACL,MAAM,QAAQ,KAAKN;IACnB,IACE,KAAKO,yBAAyB,KAAA,KAC9B,MAAM,KAAKA,wBAAwB,OACnC;KACA,KAAKA,uBAAuB;KAC5B,KAAKb,IAAI,OACP,qBAAqB,IAAI,GAAG,kBAAkB,gBAAgB,MACxD,KAAKY,qBAAqB,oEAEhC,EAAC,MAAK,CACR;IACF;GACF,OAAO;IACL,KAAKZ,IAAI,OACP,sBAAsB,IAAI,GAAG,kBAAkB,gBAAgB,oBAC/D,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EAEJ;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;EACnC,KAAKI,6BAA6B;CACpC;;;;;;CAOA,IAAI,KAAiB;EACnB,KAAKA,6BAA6B;EAClC,IAAI,KAAKC,gBACP,KAAKX,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKW,6BAA6B;EAClC,KAAKX,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,KAAKY,SAAS,KAAK,KAAK;EAExB,IACE,KAAKZ,QAAQ,OAAO,MACnB,MAAM,OAAO,YAAY,MAAM,OAAO,aAEvC,KAAKU,6BAA6B;EAEpC,KAAKG,yBAAyB,MAAM,EAAE;EAEtC,IAAI,KAAKD,SAAS,UAAU,oBAC1B,KAAKF,6BAA6B;OAC7B,IAAI,CAAC,KAAKI,mBAAmB,KAAKF,SAAS,SAAS,GAAG;GAC5D,KAAKE,kBAAkB;GACvB,KAAKC,oBAAoB,mBACvB,KAAKL,6BAA6B,CACpC;EACF;CACF;CAEA,WAAW,QAAgB;EACzB,KAAKA,6BAA6B;EAClC,KAAK,MAAM,OAAO,KAAKX,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,KAAKa,SAAS,KAAK,KAAK;EAExB,MAAM,QAAQ,KAAKI,cAAc;EACjC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKpB,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKY,yBAAyB,MAAM,EAAE;EAGtC,KAAKN,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,gBAAqC;EACnC,IAAI,KAAKQ,sBAAsB,KAAA,GAAW;GACxC,eAAe,KAAKA,iBAAiB;GACrC,KAAKA,oBAAoB,KAAA;EAC3B;EACA,KAAKD,kBAAkB;EACvB,IAAI,KAAKF,SAAS,WAAW,GAC3B,OAAO,CAAC;EAEV,MAAM,QAAQ,KAAKA;EACnB,KAAKA,WAAW,CAAC;EACjB,OAAO;CACT;CAEA,+BAA+B;EAC7B,MAAM,QAAQ,KAAKI,cAAc;EACjC,IAAI,MAAM,SAAS,GACjB,UAAU,gBAAgB,KAAKjB,QAAQ,OAAO,GAAG,KAAK;CAE1D;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKA,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKW,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKX,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.d.ts

@@ -0,0 +1,51 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Source } from '../../types/streams.ts';
+import { LitestreamController } from '../litestream/litestream-controller.ts';
+import type { BackedUpWatermark } from './backup-monitor.ts';
+/** The subset of {@link LitestreamController} used by the requester. */
+export type SyncController = Pick<LitestreamController, 'sync' | 'close'>;
+export type SyncRequesterConfig = {
+    /**
+     * Interval between sync requests. This paces how often litestream seals
+     * (and uploads) a new LTX file.
+     */
+    intervalMs: number;
+};
+/**
+ * The LitestreamSyncRequester drives litestream v5 backups and tracks which
+ * replica watermark is durably backed up, emitting it as a stream of
+ * {@link BackedUpWatermark}s.
+ *
+ * It periodically reads the replica's current watermark `W` and then asks
+ * litestream to sync and wait for the upload (`POST /sync {wait: true}`).
+ * litestream satisfies the request with a sync pass that *starts after* the
+ * request, and a pass syncs up to the end of the WAL, so a successful response
+ * means that every transaction committed before the request, including the
+ * one that wrote `W`, has been uploaded to the backup. No part of the backup
+ * needs to be read to determine this.
+ *
+ * Requests continue even when `W` is already backed up, so that changes that
+ * don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)
+ * are also backed up at this cadence rather than only at litestream's slower
+ * backstop interval. A request on an idle replica is a local no-op for
+ * litestream: there is nothing to seal or upload, and no backup reads.
+ *
+ * At most one request is outstanding at a time. Failed or timed-out requests
+ * are retried on the next tick, and nothing is published until one succeeds.
+ */
+export declare class LitestreamSyncRequester {
+    #private;
+    constructor(lc: LogContext, replicaFile: string, config: SyncRequesterConfig, litestream?: SyncController);
+    /**
+     * Starts requesting syncs and pushes backed-up watermarks to the returned
+     * stream. Must only be called once.
+     */
+    start(): Source<BackedUpWatermark>;
+    /**
+     * Requests a sync unless a request is already outstanding. Resolves when
+     * this tick's request (or the outstanding one) completes.
+     * Exported for testing.
+     */
+    readonly tick: () => Promise<void>;
+}
+//# sourceMappingURL=litestream-sync-requester.d.ts.map

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream-sync-requester.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/litestream-sync-requester.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAEnD,OAAO,EAAC,oBAAoB,EAAC,MAAM,wCAAwC,CAAC;AAE5E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAyB3D,wEAAwE;AACxE,MAAM,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC;AAE1E,MAAM,MAAM,mBAAmB,GAAG;IAChC;;;OAGG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,uBAAuB;;IAalC,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,mBAAmB,EAC3B,UAAU,GAAE,cAA0D,EAkBvE;IAED;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC,CAQjC;IAED;;;;OAIG;IACH,QAAQ,CAAC,IAAI,QAAO,OAAO,CAAC,IAAI,CAAC,CAe/B;CAyCH"}

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.js

@@ -0,0 +1,132 @@
+import { assert } from "../../../../shared/src/asserts.js";
+import { parse, valita_exports } from "../../../../shared/src/valita.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { RunningState } from "../running-state.js";
+import { Subscription } from "../../types/subscription.js";
+import { LitestreamController } from "../litestream/litestream-controller.js";
+//#region ../zero-cache/src/services/change-streamer/litestream-sync-requester.ts
+/**
+* How long litestream waits for a requested sync (and its upload) before
+* ending the request. A request that times out is simply retried on a later
+* tick: litestream only abandons the *wait*, never the sync or upload itself,
+* so this bounds request lifetime without affecting backup progress.
+*/
+var SERVER_TIMEOUT_SECONDS = 60;
+/**
+* Client-side timeout, slightly above the server timeout so that litestream,
+* rather than the client, ends a slow request.
+*/
+var CLIENT_TIMEOUT_MS = (SERVER_TIMEOUT_SECONDS + 5) * 1e3;
+var ERROR_LOG_INTERVAL_MS = 6e4;
+var localWatermarkSchema = valita_exports.object({
+	stateVersion: valita_exports.string(),
+	writeTimeMs: valita_exports.number()
+});
+/**
+* The LitestreamSyncRequester drives litestream v5 backups and tracks which
+* replica watermark is durably backed up, emitting it as a stream of
+* {@link BackedUpWatermark}s.
+*
+* It periodically reads the replica's current watermark `W` and then asks
+* litestream to sync and wait for the upload (`POST /sync {wait: true}`).
+* litestream satisfies the request with a sync pass that *starts after* the
+* request, and a pass syncs up to the end of the WAL, so a successful response
+* means that every transaction committed before the request, including the
+* one that wrote `W`, has been uploaded to the backup. No part of the backup
+* needs to be read to determine this.
+*
+* Requests continue even when `W` is already backed up, so that changes that
+* don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)
+* are also backed up at this cadence rather than only at litestream's slower
+* backstop interval. A request on an idle replica is a local no-op for
+* litestream: there is nothing to seal or upload, and no backup reads.
+*
+* At most one request is outstanding at a time. Failed or timed-out requests
+* are retried on the next tick, and nothing is published until one succeeds.
+*/
+var LitestreamSyncRequester = class {
+	#lc;
+	#state = new RunningState("litestream-sync-requester");
+	#config;
+	#litestream;
+	#readLocalWatermark;
+	#stream;
+	#timer;
+	#inFlight;
+	#backedUp;
+	#lastErrorLogMs = 0;
+	constructor(lc, replicaFile, config, litestream = new LitestreamController(lc, replicaFile)) {
+		this.#lc = lc.withContext("component", "litestream-sync-requester");
+		this.#config = config;
+		this.#litestream = litestream;
+		const db = new Database(this.#lc, replicaFile, { readonly: true });
+		this.#readLocalWatermark = db.prepare(`SELECT stateVersion, writeTimeMs FROM "_zero.replicationState"`);
+		this.#stream = Subscription.create({ cleanup: () => {
+			clearInterval(this.#timer);
+			this.#state.stop(this.#lc);
+			this.#litestream.close();
+			db.close();
+		} });
+	}
+	/**
+	* Starts requesting syncs and pushes backed-up watermarks to the returned
+	* stream. Must only be called once.
+	*/
+	start() {
+		assert(this.#timer === void 0, `Already called start()`);
+		this.#lc.info?.(`requesting litestream syncs every ${this.#config.intervalMs} ms`);
+		this.#timer = setInterval(this.tick, this.#config.intervalMs);
+		this.tick();
+		return this.#stream;
+	}
+	/**
+	* Requests a sync unless a request is already outstanding. Resolves when
+	* this tick's request (or the outstanding one) completes.
+	* Exported for testing.
+	*/
+	tick = () => {
+		if (!this.#state.shouldRun() || this.#inFlight) return this.#inFlight ?? Promise.resolve();
+		let local;
+		try {
+			local = parse(this.#readLocalWatermark.get(), localWatermarkSchema);
+		} catch (e) {
+			this.#lc.warn?.(`unable to read local watermark`, e);
+			return Promise.resolve();
+		}
+		this.#inFlight = this.#requestSync(local).finally(() => {
+			this.#inFlight = void 0;
+		});
+		return this.#inFlight;
+	};
+	async #requestSync(local) {
+		try {
+			const result = await this.#litestream.sync({
+				wait: true,
+				timeoutMs: CLIENT_TIMEOUT_MS,
+				serverTimeoutSeconds: SERVER_TIMEOUT_SECONDS
+			}, this.#state.signal);
+			this.#publish(local, Date.now());
+			this.#lc.debug?.(`backed up watermark ${local.stateVersion}`, result);
+		} catch (e) {
+			if (!this.#state.shouldRun()) return;
+			const now = Date.now();
+			if (now - this.#lastErrorLogMs >= ERROR_LOG_INTERVAL_MS) {
+				this.#lastErrorLogMs = now;
+				this.#lc.info?.(`backup of watermark ${local.stateVersion} not yet confirmed. retrying`, e);
+			}
+		}
+	}
+	#publish({ stateVersion, writeTimeMs }, backupTimeMs) {
+		if (this.#backedUp !== void 0 && stateVersion <= this.#backedUp) return;
+		this.#backedUp = stateVersion;
+		this.#stream.push({
+			watermark: stateVersion,
+			writeTimeMs,
+			backupTimeMs
+		});
+	}
+};
+//#endregion
+export { LitestreamSyncRequester };
+
+//# sourceMappingURL=litestream-sync-requester.js.map

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream-sync-requester.js","names":["#lc","#state","#config","#litestream","#readLocalWatermark","#stream","#timer","#inFlight","#requestSync","#publish","#lastErrorLogMs","#backedUp"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream-sync-requester.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {LitestreamController} from '../litestream/litestream-controller.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\n/**\n * How long litestream waits for a requested sync (and its upload) before\n * ending the request. A request that times out is simply retried on a later\n * tick: litestream only abandons the *wait*, never the sync or upload itself,\n * so this bounds request lifetime without affecting backup progress.\n */\nconst SERVER_TIMEOUT_SECONDS = 60;\n\n/**\n * Client-side timeout, slightly above the server timeout so that litestream,\n * rather than the client, ends a slow request.\n */\nconst CLIENT_TIMEOUT_MS = (SERVER_TIMEOUT_SECONDS + 5) * 1000;\n\nconst ERROR_LOG_INTERVAL_MS = 60_000;\n\nconst localWatermarkSchema = v.object({\n  stateVersion: v.string(),\n  writeTimeMs: v.number(),\n});\n\ntype LocalWatermark = v.Infer<typeof localWatermarkSchema>;\n\n/** The subset of {@link LitestreamController} used by the requester. */\nexport type SyncController = Pick<LitestreamController, 'sync' | 'close'>;\n\nexport type SyncRequesterConfig = {\n  /**\n   * Interval between sync requests. This paces how often litestream seals\n   * (and uploads) a new LTX file.\n   */\n  intervalMs: number;\n};\n\n/**\n * The LitestreamSyncRequester drives litestream v5 backups and tracks which\n * replica watermark is durably backed up, emitting it as a stream of\n * {@link BackedUpWatermark}s.\n *\n * It periodically reads the replica's current watermark `W` and then asks\n * litestream to sync and wait for the upload (`POST /sync {wait: true}`).\n * litestream satisfies the request with a sync pass that *starts after* the\n * request, and a pass syncs up to the end of the WAL, so a successful response\n * means that every transaction committed before the request, including the\n * one that wrote `W`, has been uploaded to the backup. No part of the backup\n * needs to be read to determine this.\n *\n * Requests continue even when `W` is already backed up, so that changes that\n * don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)\n * are also backed up at this cadence rather than only at litestream's slower\n * backstop interval. A request on an idle replica is a local no-op for\n * litestream: there is nothing to seal or upload, and no backup reads.\n *\n * At most one request is outstanding at a time. Failed or timed-out requests\n * are retried on the next tick, and nothing is published until one succeeds.\n */\nexport class LitestreamSyncRequester {\n  readonly #lc: LogContext;\n  readonly #state = new RunningState('litestream-sync-requester');\n  readonly #config: SyncRequesterConfig;\n  readonly #litestream: SyncController;\n  readonly #readLocalWatermark: Statement;\n  readonly #stream: Subscription<BackedUpWatermark>;\n\n  #timer: NodeJS.Timeout | undefined;\n  #inFlight: Promise<void> | undefined;\n  #backedUp: string | undefined;\n  #lastErrorLogMs = 0;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    config: SyncRequesterConfig,\n    litestream: SyncController = new LitestreamController(lc, replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-sync-requester');\n    this.#config = config;\n    this.#litestream = litestream;\n\n    const db = new Database(this.#lc, replicaFile, {readonly: true});\n    this.#readLocalWatermark = db.prepare(\n      `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n    );\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#timer);\n        this.#state.stop(this.#lc);\n        this.#litestream.close();\n        db.close();\n      },\n    });\n  }\n\n  /**\n   * Starts requesting syncs and pushes backed-up watermarks to the returned\n   * stream. Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#timer === undefined, `Already called start()`);\n    this.#lc.info?.(\n      `requesting litestream syncs every ${this.#config.intervalMs} ms`,\n    );\n    this.#timer = setInterval(this.tick, this.#config.intervalMs);\n    void this.tick();\n    return this.#stream;\n  }\n\n  /**\n   * Requests a sync unless a request is already outstanding. Resolves when\n   * this tick's request (or the outstanding one) completes.\n   * Exported for testing.\n   */\n  readonly tick = (): Promise<void> => {\n    if (!this.#state.shouldRun() || this.#inFlight) {\n      return this.#inFlight ?? Promise.resolve();\n    }\n    let local: LocalWatermark;\n    try {\n      local = v.parse(this.#readLocalWatermark.get(), localWatermarkSchema);\n    } catch (e) {\n      this.#lc.warn?.(`unable to read local watermark`, e);\n      return Promise.resolve();\n    }\n    this.#inFlight = this.#requestSync(local).finally(() => {\n      this.#inFlight = undefined;\n    });\n    return this.#inFlight;\n  };\n\n  async #requestSync(local: LocalWatermark) {\n    // `local` must be read before the request is issued: litestream only\n    // guarantees that transactions committed before the request are synced.\n    try {\n      const result = await this.#litestream.sync(\n        {\n          wait: true,\n          timeoutMs: CLIENT_TIMEOUT_MS,\n          serverTimeoutSeconds: SERVER_TIMEOUT_SECONDS,\n        },\n        this.#state.signal,\n      );\n      this.#publish(local, Date.now());\n      this.#lc.debug?.(`backed up watermark ${local.stateVersion}`, result);\n    } catch (e) {\n      if (!this.#state.shouldRun()) {\n        return; // shutting down\n      }\n      // Expected while litestream is starting up or busy with a long sync or\n      // upload (e.g. the initial snapshot of a large replica); the request is\n      // retried on the next tick.\n      const now = Date.now();\n      if (now - this.#lastErrorLogMs >= ERROR_LOG_INTERVAL_MS) {\n        this.#lastErrorLogMs = now;\n        this.#lc.info?.(\n          `backup of watermark ${local.stateVersion} not yet confirmed. retrying`,\n          e,\n        );\n      }\n    }\n  }\n\n  #publish({stateVersion, writeTimeMs}: LocalWatermark, backupTimeMs: number) {\n    if (this.#backedUp !== undefined && stateVersion <= this.#backedUp) {\n      return;\n    }\n    this.#backedUp = stateVersion;\n    this.#stream.push({watermark: stateVersion, writeTimeMs, backupTimeMs});\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAM,yBAAyB;;;;;AAM/B,IAAM,qBAAqB,yBAAyB,KAAK;AAEzD,IAAM,wBAAwB;AAE9B,IAAM,uBAAuB,eAAE,OAAO;CACpC,cAAc,eAAE,OAAO;CACvB,aAAa,eAAE,OAAO;AACxB,CAAC;;;;;;;;;;;;;;;;;;;;;;;AAqCD,IAAa,0BAAb,MAAqC;CACnC;CACA,SAAkB,IAAI,aAAa,2BAA2B;CAC9D;CACA;CACA;CACA;CAEA;CACA;CACA;CACA,kBAAkB;CAElB,YACE,IACA,aACA,QACA,aAA6B,IAAI,qBAAqB,IAAI,WAAW,GACrE;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAKE,UAAU;EACf,KAAKC,cAAc;EAEnB,MAAM,KAAK,IAAI,SAAS,KAAKH,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,KAAKI,sBAAsB,GAAG,QAC5B,gEACF;EACA,KAAKC,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKC,MAAM;GACzB,KAAKL,OAAO,KAAK,KAAKD,GAAG;GACzB,KAAKG,YAAY,MAAM;GACvB,GAAG,MAAM;EACX,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKG,WAAW,KAAA,GAAW,wBAAwB;EAC1D,KAAKN,IAAI,OACP,qCAAqC,KAAKE,QAAQ,WAAW,IAC/D;EACA,KAAKI,SAAS,YAAY,KAAK,MAAM,KAAKJ,QAAQ,UAAU;EAC5D,KAAU,KAAK;EACf,OAAO,KAAKG;CACd;;;;;;CAOA,aAAqC;EACnC,IAAI,CAAC,KAAKJ,OAAO,UAAU,KAAK,KAAKM,WACnC,OAAO,KAAKA,aAAa,QAAQ,QAAQ;EAE3C,IAAI;EACJ,IAAI;GACF,QAAQ,MAAQ,KAAKH,oBAAoB,IAAI,GAAG,oBAAoB;EACtE,SAAS,GAAG;GACV,KAAKJ,IAAI,OAAO,kCAAkC,CAAC;GACnD,OAAO,QAAQ,QAAQ;EACzB;EACA,KAAKO,YAAY,KAAKC,aAAa,KAAK,EAAE,cAAc;GACtD,KAAKD,YAAY,KAAA;EACnB,CAAC;EACD,OAAO,KAAKA;CACd;CAEA,MAAMC,aAAa,OAAuB;EAGxC,IAAI;GACF,MAAM,SAAS,MAAM,KAAKL,YAAY,KACpC;IACE,MAAM;IACN,WAAW;IACX,sBAAsB;GACxB,GACA,KAAKF,OAAO,MACd;GACA,KAAKQ,SAAS,OAAO,KAAK,IAAI,CAAC;GAC/B,KAAKT,IAAI,QAAQ,uBAAuB,MAAM,gBAAgB,MAAM;EACtE,SAAS,GAAG;GACV,IAAI,CAAC,KAAKC,OAAO,UAAU,GACzB;GAKF,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,MAAM,KAAKS,mBAAmB,uBAAuB;IACvD,KAAKA,kBAAkB;IACvB,KAAKV,IAAI,OACP,uBAAuB,MAAM,aAAa,+BAC1C,CACF;GACF;EACF;CACF;CAEA,SAAS,EAAC,cAAc,eAA8B,cAAsB;EAC1E,IAAI,KAAKW,cAAc,KAAA,KAAa,gBAAgB,KAAKA,WACvD;EAEF,KAAKA,YAAY;EACjB,KAAKN,QAAQ,KAAK;GAAC,WAAW;GAAc;GAAa;EAAY,CAAC;CACxE;AACF"}

package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts

@@ -48,7 +48,7 @@ interface PurgeLock {
 }
 export declare function setupCDCTables(lc: LogContext, db: postgres.TransactionSql, shard: ShardID): Promise<void>;
 export declare function markResetRequired(sql: PostgresDB, shard: ShardID): Promise<void>;
-export declare function ensureReplicationConfig(lc: LogContext, db: PostgresDB, subscriptionState: Pick<SubscriptionState, 'publications' | 'replicaVersion' | 'watermark'>, shard: ShardID, autoReset: boolean, purgeLock?: PurgeLock, setTimeoutFn?: typeof setTimeout): Promise<void>;
+export declare function ensureReplicationConfig(lc: LogContext, db: PostgresDB, subscriptionState: Pick<SubscriptionState, 'publications' | 'replicaVersion' | 'watermark'>, shard: ShardID, autoReset: boolean, purgeLock?: PurgeLock, setTimeoutFn?: typeof setTimeout, pgChangeLogEnabled?: boolean): Promise<void>;
 export declare const CHANGE_STREAMER_APP_NAME = "zero-change-streamer";
 export declare class AutoResetSignal extends AbortError {
     readonly name = "AutoResetSignal";

package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EACV,UAAU,EACV,MAAM,EACN,aAAa,EACd,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAmBF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAaF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,UAiBlD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAYF,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,IAAI,CACrB,iBAAiB,EACjB,cAAc,GAAG,gBAAgB,GAAG,WAAW,CAChD,EACD,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,EAClB,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,iBA4H7C;AAMD,eAAO,MAAM,wBAAwB,yBAAyB,CAAC;AAE/D,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,iBA4Cf"}
+{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EACV,UAAU,EACV,MAAM,EACN,aAAa,EACd,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAmBF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAaF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,UAiBlD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAYF,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,IAAI,CACrB,iBAAiB,EACjB,cAAc,GAAG,gBAAgB,GAAG,WAAW,CAChD,EACD,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,EAClB,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,EAC5C,kBAAkB,UAAO,iBA8I1B;AAMD,eAAO,MAAM,wBAAwB,yBAAyB,CAAC;AAE/D,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,iBA4Cf"}

package/out/zero-cache/src/services/change-streamer/schema/tables.js

@@ -77,7 +77,7 @@ async function markResetRequired(sql, shard) {
   UPDATE ${sql(cdcSchema(shard))}."replicationConfig"
     SET "resetRequired" = true`;
 }
-async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoReset, purgeLock, setTimeoutFn = setTimeout) {
+async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoReset, purgeLock, setTimeoutFn = setTimeout, pgChangeLogEnabled = true) {
 	const { publications, replicaVersion, watermark } = subscriptionState;
 	const replicaConfig = {
 		publications,
@@ -102,12 +102,12 @@ async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoRes
 				lc.info?.(`Data in cdc tables @${replicaVersion} is incompatible with replica @${replicaConfig.replicaVersion}. Clearing tables.`);
 				purgeLock?.release();
 				needsTruncate = true;
-				stmts.push(sql`TRUNCATE TABLE ${sql(schema)}."replicationState"`, sql`TRUNCATE TABLE ${sql(schema)}."changeLog"`, sql`TRUNCATE TABLE ${sql(schema)}."replicationConfig"`, sql`TRUNCATE TABLE ${sql(schema)}."tableMetadata"`, sql`TRUNCATE TABLE ${sql(schema)}."backfilling"`);
+				stmts.push(sql`TRUNCATE TABLE ${sql(schema)}."replicationState"`, ...pgChangeLogEnabled ? [sql`TRUNCATE TABLE ${sql(schema)}."changeLog"`] : [], sql`TRUNCATE TABLE ${sql(schema)}."replicationConfig"`, sql`TRUNCATE TABLE ${sql(schema)}."tableMetadata"`, sql`TRUNCATE TABLE ${sql(schema)}."backfilling"`);
 			}
 		}
 		if (results.length === 0 || needsTruncate) {
 			const watermark = replicaConfig.replicaVersion;
-			const initialTx = [{
+			const initialTx = pgChangeLogEnabled ? [{
 				watermark,
 				pos: 0,
 				change: { tag: "begin" }
@@ -115,17 +115,28 @@ async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoRes
 				watermark,
 				pos: 1,
 				change: { tag: "commit" }
-			}];
+			}] : [];
 			stmts.push(sql`INSERT INTO ${sql(schema)}."replicationConfig" ${sql(replicaConfig)}`, sql`INSERT INTO ${sql(schema)}."replicationState"  ${sql(replicationState)} 
               ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`, ...initialTx.map((change) => sql`INSERT INTO ${sql(schema)}."changeLog" ${sql(change)}`));
 			if (needsTruncate) {
-				const timer = setTimeoutFn(async () => {
-					lc.info?.("ensureReplicationConfig blocked, terminating lock holders");
-					await terminateChangeDBLockHolders(lc, db, shard);
-				}, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);
+				let done = false;
+				let timer;
+				const scheduleTerminate = () => {
+					timer = setTimeoutFn(async () => {
+						lc.info?.("ensureReplicationConfig blocked, terminating lock holders");
+						try {
+							await terminateChangeDBLockHolders(lc, db, shard);
+						} catch (e) {
+							lc.warn?.("error terminating lock holders", e);
+						}
+						if (!done) scheduleTerminate();
+					}, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);
+				};
+				scheduleTerminate();
 				try {
 					return await Promise.all(stmts);
 				} finally {
+					done = true;
 					clearTimeout(timer);
 				}
 			}

package/out/zero-cache/src/services/change-streamer/schema/tables.js.map

@@ -1 +1 @@
-{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: Pick<\n    SubscriptionState,\n    'publications' | 'replicaVersion' | 'watermark'\n  >,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n) {\n  const {publications, replicaVersion, watermark} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const stmts: PendingQuery<Row[]>[] = [];\n    let needsTruncate = false;\n    const results = await sql<\n      {\n        replicaVersion: string;\n        publications: string[];\n        resetRequired: boolean | null;\n      }[]\n    > /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(schema)}.\"replicationConfig\"`;\n\n    if (results.length) {\n      const {replicaVersion, publications} = results[0];\n      if (\n        replicaVersion !== replicaConfig.replicaVersion ||\n        !equals(new Set(publications), new Set(replicaConfig.publications))\n      ) {\n        if (replicaConfig.replicaVersion !== watermark) {\n          throw new AutoResetSignal(\n            `Cannot reset change db@${replicaVersion} to ` +\n              `service replica@${replicaConfig.replicaVersion} ` +\n              `from watermark ${watermark}`,\n          );\n        }\n        lc.info?.(\n          `Data in cdc tables @${replicaVersion} is incompatible ` +\n            `with replica @${replicaConfig.replicaVersion}. Clearing tables.`,\n        );\n        // Release any purge lock held by the caller; the changeLog is\n        // incompatible with the replica and needs to be truncated.\n        void purgeLock?.release();\n\n        // Note: The order of TRUNCATE matters. The replicationState table is\n        //       truncated before the changeLog table, in order to acquire\n        //       (exclusive) locks on the tables in the same order that the\n        //       storer.ts acquires locks when writing the changeLog, namely:\n        //\n        // 1. SELECT ... FROM replicationState FOR UPDATE;\n        // 2. INSERT INTO changeLog ...;\n        //    ...\n        // n. UPDATE replicationState ...;\n        needsTruncate = true;\n        stmts.push(\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n        );\n      }\n    }\n    // Initialize (or re-initialize TRUNCATED) tables\n    if (results.length === 0 || needsTruncate) {\n      // The storer uses the earliest changeLog entry as the safe watermark\n      // from which subscribers can be resumed. These initial entries ensure\n      // that subscribers can start from a freshly synced replica, even if\n      // new changes have been replicated and not purged from the changeLog.\n      //\n      // TODO: Replace this with an explicit `firstWatermark` column in the\n      //       change db.\n      const watermark = replicaConfig.replicaVersion;\n      const initialTx: FullChangeLogEntry[] = [\n        {watermark, pos: 0, change: {tag: 'begin'}},\n        {watermark, pos: 1, change: {tag: 'commit'}},\n      ];\n\n      stmts.push(\n        sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n        sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n              ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n        ...initialTx.map(\n          change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n        ),\n      );\n\n      if (needsTruncate) {\n        // The TRUNCATE statements require ACCESS EXCLUSIVE locks, which may\n        // be blocked by old storer catchup reads. Race against a timeout\n        // that terminates the blocking backends if the TRUNCATE takes too\n        // long.\n        const timer = setTimeoutFn(async () => {\n          lc.info?.(\n            'ensureReplicationConfig blocked, terminating lock holders',\n          );\n          await terminateChangeDBLockHolders(lc, db, shard);\n        }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n        try {\n          return await Promise.all(stmts);\n        } finally {\n          clearTimeout(timer);\n        }\n      }\n      return Promise.all(stmts);\n    }\n\n    const {resetRequired} = results[0];\n    if (resetRequired) {\n      if (autoReset) {\n        throw new AutoResetSignal('reset required by replication stream');\n      }\n      lc.error?.(\n        '\\n\\n\\n' +\n          'Reset required but --auto-reset is not enabled.\\n' +\n          'This can happen for upstream databases that do not support event triggers.\\n' +\n          'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n          '\\n\\n\\n',\n      );\n    }\n\n    return [];\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nexport async function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,sBACvC,GAAG;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;AAEA,eAAsB,wBACpB,IACA,IACA,mBAIA,OACA,WACA,WACA,eAAkC,YAClC;CACA,MAAM,EAAC,cAAc,gBAAgB,cAAa;CAClD,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,QAA+B,CAAC;EACtC,IAAI,gBAAgB;EACpB,MAAM,UAAU,MAAM,GAMZ;;aAED,IAAI,MAAM,EAAE;EAErB,IAAI,QAAQ,QAAQ;GAClB,MAAM,EAAC,gBAAgB,iBAAgB,QAAQ;GAC/C,IACE,mBAAmB,cAAc,kBACjC,CAAC,OAAO,IAAI,IAAI,YAAY,GAAG,IAAI,IAAI,cAAc,YAAY,CAAC,GAClE;IACA,IAAI,cAAc,mBAAmB,WACnC,MAAM,IAAI,gBACR,0BAA0B,eAAe,sBACpB,cAAc,eAAe,kBAC9B,WACtB;IAEF,GAAG,OACD,uBAAuB,eAAe,iCACnB,cAAc,eAAe,mBAClD;IAGA,WAAgB,QAAQ;IAWxB,gBAAgB;IAChB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;GACF;EACF;EAEA,IAAI,QAAQ,WAAW,KAAK,eAAe;GAQzC,MAAM,YAAY,cAAc;GAChC,MAAM,YAAkC,CACtC;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,QAAO;GAAC,GAC1C;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,SAAQ;GAAC,CAC7C;GAEA,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;iDAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;GAEA,IAAI,eAAe;IAKjB,MAAM,QAAQ,aAAa,YAAY;KACrC,GAAG,OACD,2DACF;KACA,MAAM,6BAA6B,IAAI,IAAI,KAAK;IAClD,GAAG,gCAAgC;IACnC,IAAI;KACF,OAAO,MAAM,QAAQ,IAAI,KAAK;IAChC,UAAU;KACR,aAAa,KAAK;IACpB;GACF;GACA,OAAO,QAAQ,IAAI,KAAK;EAC1B;EAEA,MAAM,EAAC,kBAAiB,QAAQ;EAChC,IAAI,eAAe;GACjB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;GAElE,GAAG,QACD,oOAKF;EACF;EAEA,OAAO,CAAC;CACV,CAAC;AACH;AAIA,IAAM,mCAAmC;AAEzC,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}
+{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: Pick<\n    SubscriptionState,\n    'publications' | 'replicaVersion' | 'watermark'\n  >,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n  pgChangeLogEnabled = true,\n) {\n  const {publications, replicaVersion, watermark} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const stmts: PendingQuery<Row[]>[] = [];\n    let needsTruncate = false;\n    const results = await sql<\n      {\n        replicaVersion: string;\n        publications: string[];\n        resetRequired: boolean | null;\n      }[]\n    > /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(schema)}.\"replicationConfig\"`;\n\n    if (results.length) {\n      const {replicaVersion, publications} = results[0];\n      if (\n        replicaVersion !== replicaConfig.replicaVersion ||\n        !equals(new Set(publications), new Set(replicaConfig.publications))\n      ) {\n        if (replicaConfig.replicaVersion !== watermark) {\n          throw new AutoResetSignal(\n            `Cannot reset change db@${replicaVersion} to ` +\n              `service replica@${replicaConfig.replicaVersion} ` +\n              `from watermark ${watermark}`,\n          );\n        }\n        lc.info?.(\n          `Data in cdc tables @${replicaVersion} is incompatible ` +\n            `with replica @${replicaConfig.replicaVersion}. Clearing tables.`,\n        );\n        // Release any purge lock held by the caller; the changeLog is\n        // incompatible with the replica and needs to be truncated.\n        void purgeLock?.release();\n\n        // Note: The order of TRUNCATE matters. The replicationState table is\n        //       truncated before the changeLog table, in order to acquire\n        //       (exclusive) locks on the tables in the same order that the\n        //       storer.ts acquires locks when writing the changeLog, namely:\n        //\n        // 1. SELECT ... FROM replicationState FOR UPDATE;\n        // 2. INSERT INTO changeLog ...;\n        //    ...\n        // n. UPDATE replicationState ...;\n        needsTruncate = true;\n        stmts.push(\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n          ...(pgChangeLogEnabled\n            ? [sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`]\n            : []),\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n        );\n      }\n    }\n    // Initialize (or re-initialize TRUNCATED) tables\n    if (results.length === 0 || needsTruncate) {\n      // When enabled, the PG storer uses the earliest changeLog entry as the\n      // safe watermark from which subscribers can be resumed. These initial\n      // entries ensure that subscribers can start from a freshly synced\n      // replica, even if new changes have been replicated and not purged.\n      //\n      // TODO: Replace this with an explicit `firstWatermark` column in the\n      //       change db.\n      const watermark = replicaConfig.replicaVersion;\n      const initialTx: FullChangeLogEntry[] = pgChangeLogEnabled\n        ? [\n            {watermark, pos: 0, change: {tag: 'begin'}},\n            {watermark, pos: 1, change: {tag: 'commit'}},\n          ]\n        : [];\n\n      stmts.push(\n        sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n        sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n              ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n        ...initialTx.map(\n          change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n        ),\n      );\n\n      if (needsTruncate) {\n        // The TRUNCATE statements require ACCESS EXCLUSIVE locks, which may\n        // be blocked by old storer catchup reads. Race against a timeout\n        // that terminates the blocking backends if the TRUNCATE takes too\n        // long. The check is repeated until the statements complete, since\n        // the TRUNCATE may not yet be waiting on a lock when the timer fires.\n        let done = false;\n        let timer: ReturnType<typeof setTimeout> | undefined;\n        const scheduleTerminate = () => {\n          timer = setTimeoutFn(async () => {\n            lc.info?.(\n              'ensureReplicationConfig blocked, terminating lock holders',\n            );\n            try {\n              await terminateChangeDBLockHolders(lc, db, shard);\n            } catch (e) {\n              lc.warn?.('error terminating lock holders', e);\n            }\n            if (!done) {\n              scheduleTerminate();\n            }\n          }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n        };\n        scheduleTerminate();\n        try {\n          return await Promise.all(stmts);\n        } finally {\n          done = true;\n          clearTimeout(timer);\n        }\n      }\n      return Promise.all(stmts);\n    }\n\n    const {resetRequired} = results[0];\n    if (resetRequired) {\n      if (autoReset) {\n        throw new AutoResetSignal('reset required by replication stream');\n      }\n      lc.error?.(\n        '\\n\\n\\n' +\n          'Reset required but --auto-reset is not enabled.\\n' +\n          'This can happen for upstream databases that do not support event triggers.\\n' +\n          'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n          '\\n\\n\\n',\n      );\n    }\n\n    return [];\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nexport async function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,sBACvC,GAAG;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;AAEA,eAAsB,wBACpB,IACA,IACA,mBAIA,OACA,WACA,WACA,eAAkC,YAClC,qBAAqB,MACrB;CACA,MAAM,EAAC,cAAc,gBAAgB,cAAa;CAClD,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,QAA+B,CAAC;EACtC,IAAI,gBAAgB;EACpB,MAAM,UAAU,MAAM,GAMZ;;aAED,IAAI,MAAM,EAAE;EAErB,IAAI,QAAQ,QAAQ;GAClB,MAAM,EAAC,gBAAgB,iBAAgB,QAAQ;GAC/C,IACE,mBAAmB,cAAc,kBACjC,CAAC,OAAO,IAAI,IAAI,YAAY,GAAG,IAAI,IAAI,cAAc,YAAY,CAAC,GAClE;IACA,IAAI,cAAc,mBAAmB,WACnC,MAAM,IAAI,gBACR,0BAA0B,eAAe,sBACpB,cAAc,eAAe,kBAC9B,WACtB;IAEF,GAAG,OACD,uBAAuB,eAAe,iCACnB,cAAc,eAAe,mBAClD;IAGA,WAAgB,QAAQ;IAWxB,gBAAgB;IAChB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAI,qBACA,CAAC,GAAG,kBAAkB,IAAI,MAAM,EAAE,aAAa,IAC/C,CAAC,GACL,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;GACF;EACF;EAEA,IAAI,QAAQ,WAAW,KAAK,eAAe;GAQzC,MAAM,YAAY,cAAc;GAChC,MAAM,YAAkC,qBACpC,CACE;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,QAAO;GAAC,GAC1C;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,SAAQ;GAAC,CAC7C,IACA,CAAC;GAEL,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;iDAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;GAEA,IAAI,eAAe;IAMjB,IAAI,OAAO;IACX,IAAI;IACJ,MAAM,0BAA0B;KAC9B,QAAQ,aAAa,YAAY;MAC/B,GAAG,OACD,2DACF;MACA,IAAI;OACF,MAAM,6BAA6B,IAAI,IAAI,KAAK;MAClD,SAAS,GAAG;OACV,GAAG,OAAO,kCAAkC,CAAC;MAC/C;MACA,IAAI,CAAC,MACH,kBAAkB;KAEtB,GAAG,gCAAgC;IACrC;IACA,kBAAkB;IAClB,IAAI;KACF,OAAO,MAAM,QAAQ,IAAI,KAAK;IAChC,UAAU;KACR,OAAO;KACP,aAAa,KAAK;IACpB;GACF;GACA,OAAO,QAAQ,IAAI,KAAK;EAC1B;EAEA,MAAM,EAAC,kBAAiB,QAAQ;EAChC,IAAI,eAAe;GACjB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;GAElE,GAAG,QACD,oOAKF;EACF;EAEA,OAAO,CAAC;CACV,CAAC;AACH;AAIA,IAAM,mCAAmC;AAEzC,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}