@rocicorp/zero 1.12.0-canary.5 → 1.12.0-canary.7

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.js

@@ -1,5 +1,6 @@
 import { assert } from "../../../../shared/src/asserts.js";
 import { parse, valita_exports } from "../../../../shared/src/valita.js";
+import { must } from "../../../../shared/src/must.js";
 import { Database } from "../../../../zqlite/src/db.js";
 import { RunningState } from "../running-state.js";
 import { Subscription } from "../../types/subscription.js";
@@ -55,16 +56,23 @@ var localWatermarkSchema = valita_exports.object({
 * are retried with a backoff that starts short and, once it grows to the
 * interval, defers to the regular ticks. Nothing is published until a request
 * succeeds.
+*
+* A sync through a specific watermark can also be requested out of band (see
+* {@link requestSync}), e.g. when a replica fork waits on its backup.
 */
 var LitestreamSyncRequester = class {
 	#lc;
 	#state = new RunningState("litestream-sync-requester");
 	#config;
 	#litestream;
-	#readLocalWatermark;
+	#replicaFile;
 	#stream;
+	#db;
+	#readLocalWatermark;
 	#timer;
 	#retryTimer;
+	#expedited;
+	#expediteTimer;
 	#retryDelayMs = MIN_RETRY_DELAY_MS;
 	#inFlight;
 	#backedUp;
@@ -73,22 +81,27 @@ var LitestreamSyncRequester = class {
 		this.#lc = lc.withContext("component", "litestream-sync-requester");
 		this.#config = config;
 		this.#litestream = litestream;
-		const db = new Database(this.#lc, replicaFile, { readonly: true });
-		this.#readLocalWatermark = db.prepare(`SELECT stateVersion, writeTimeMs FROM "_zero.replicationState"`);
+		this.#replicaFile = replicaFile;
 		this.#stream = Subscription.create({ cleanup: () => {
 			clearInterval(this.#timer);
 			clearTimeout(this.#retryTimer);
+			clearTimeout(this.#expediteTimer);
 			this.#state.stop(this.#lc);
 			this.#litestream.close();
-			db.close();
+			this.#db?.close();
 		} });
 	}
 	/**
 	* Starts requesting syncs and pushes backed-up watermarks to the returned
 	* stream. Must only be called once.
+	*
+	* The replica is only opened here, so that a requester that is never
+	* started holds no resources.
 	*/
 	start() {
 		assert(this.#timer === void 0, `Already called start()`);
+		this.#db = new Database(this.#lc, this.#replicaFile, { readonly: true });
+		this.#readLocalWatermark = this.#db.prepare(`SELECT stateVersion, writeTimeMs FROM "_zero.replicationState"`);
 		this.#lc.info?.(`requesting litestream syncs every ${this.#config.intervalMs} ms`);
 		this.#timer = setInterval(this.tick, this.#config.intervalMs);
 		this.tick();
@@ -101,18 +114,74 @@ var LitestreamSyncRequester = class {
 	*/
 	tick = () => {
 		if (!this.#state.shouldRun() || this.#inFlight) return this.#inFlight ?? Promise.resolve();
-		let local;
-		try {
-			local = parse(this.#readLocalWatermark.get(), localWatermarkSchema);
-		} catch (e) {
-			this.#lc.warn?.(`unable to read local watermark`, e);
-			return Promise.resolve();
-		}
+		const local = this.#readLocal();
+		if (!local) return Promise.resolve();
 		this.#inFlight = this.#requestSync(local).finally(() => {
 			this.#inFlight = void 0;
 		});
 		return this.#inFlight;
 	};
+	#readLocal() {
+		try {
+			return parse(must(this.#readLocalWatermark, `start() was not called`).get(), localWatermarkSchema);
+		} catch (e) {
+			this.#lc.warn?.(`unable to read local watermark`, e);
+			return;
+		}
+	}
+	/**
+	* Requests a sync through `watermark` as soon as the local replica has
+	* reached it, rather than at the next regular tick.
+	*
+	* The replica is polled every {@link EXPEDITED_POLL_INTERVAL_MS} until its
+	* watermark reaches the target, after which a sync is requested
+	* immediately (or once the outstanding request, which may predate the
+	* target, completes). A failed request is retried with the usual backoff.
+	* Concurrent requests are coalesced into one, for the latest target.
+	*
+	* If the replica does not reach the target within the configured interval,
+	* the request is dropped, as the regular ticks then cover it anyway.
+	*/
+	requestSync(watermark) {
+		if (!this.#state.shouldRun() || this.#backedUp !== void 0 && this.#backedUp >= watermark) return;
+		const prev = this.#expedited?.target;
+		this.#expedited = {
+			target: prev !== void 0 && prev > watermark ? prev : watermark,
+			deadlineMs: Date.now() + this.#config.intervalMs
+		};
+		this.#scheduleExpedite(0);
+	}
+	#scheduleExpedite(delayMs) {
+		clearTimeout(this.#expediteTimer);
+		this.#expediteTimer = setTimeout(this.#expedite, delayMs);
+	}
+	#expedite = () => {
+		this.#expediteTimer = void 0;
+		const expedited = this.#expedited;
+		if (!expedited || !this.#state.shouldRun()) return;
+		const { target, deadlineMs } = expedited;
+		if (this.#backedUp !== void 0 && this.#backedUp >= target) {
+			this.#expedited = void 0;
+			return;
+		}
+		if (Date.now() >= deadlineMs) {
+			this.#lc.info?.(`replica did not reach ${target} within ${this.#config.intervalMs} ms. Deferring its backup to the regular syncs.`);
+			this.#expedited = void 0;
+			return;
+		}
+		if (this.#inFlight) {
+			this.#inFlight.finally(() => this.#scheduleExpedite(0));
+			return;
+		}
+		const local = this.#readLocal();
+		if (!local || local.stateVersion < target) {
+			this.#scheduleExpedite(50);
+			return;
+		}
+		this.#expedited = void 0;
+		this.#lc.info?.(`requesting sync through ${target}`);
+		this.tick();
+	};
 	async #requestSync(local) {
 		try {
 			const result = await this.#litestream.sync({

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.js.map

@@ -1 +1 @@
-{"version":3,"file":"litestream-sync-requester.js","names":["v","v.parse"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream-sync-requester.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {LitestreamController} from '../litestream/litestream-controller.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\n/**\n * How long litestream waits for a requested sync (and its upload) before\n * ending the request. A request that times out is simply retried on a later\n * tick: litestream only abandons the *wait*, never the sync or upload itself,\n * so this bounds request lifetime without affecting backup progress.\n */\nconst SERVER_TIMEOUT_SECONDS = 60;\n\n/**\n * Client-side timeout, slightly above the server timeout so that litestream,\n * rather than the client, ends a slow request.\n */\nconst CLIENT_TIMEOUT_MS = (SERVER_TIMEOUT_SECONDS + 5) * 1000;\n\nconst ERROR_LOG_INTERVAL_MS = 60_000;\n\n/**\n * Initial delay before retrying a failed request, doubled on each consecutive\n * failure until it reaches the configured interval, after which the regular\n * ticks alone retry, so that a busy litestream sees no more requests than the\n * interval allows. The first request is typically issued before litestream's\n * control socket is listening; retrying quickly (rather than after a full\n * interval) keeps the initial backup confirmation, which gates readiness, from\n * waiting out an interval.\n */\nconst MIN_RETRY_DELAY_MS = 250;\n\nconst localWatermarkSchema = v.object({\n  stateVersion: v.string(),\n  writeTimeMs: v.number(),\n});\n\ntype LocalWatermark = v.Infer<typeof localWatermarkSchema>;\n\n/** The subset of {@link LitestreamController} used by the requester. */\nexport type SyncController = Pick<LitestreamController, 'sync' | 'close'>;\n\nexport type SyncRequesterConfig = {\n  /**\n   * Interval between sync requests. This paces how often litestream seals\n   * (and uploads) a new LTX file.\n   */\n  intervalMs: number;\n};\n\n/**\n * The LitestreamSyncRequester drives litestream v5 backups and tracks which\n * replica watermark is durably backed up, emitting it as a stream of\n * {@link BackedUpWatermark}s.\n *\n * It periodically reads the replica's current watermark `W` and then asks\n * litestream to sync and wait for the upload (`POST /sync {wait: true}`).\n * litestream satisfies the request with a sync pass that *starts after* the\n * request, and a pass syncs up to the end of the WAL, so a successful response\n * means that every transaction committed before the request, including the\n * one that wrote `W`, has been uploaded to the backup. No part of the backup\n * needs to be read to determine this.\n *\n * Requests continue even when `W` is already backed up, so that changes that\n * don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)\n * are also backed up at this cadence rather than only at litestream's slower\n * backstop interval. A request on an idle replica is a local no-op for\n * litestream: there is nothing to seal or upload, and no backup reads.\n *\n * At most one request is outstanding at a time. Failed or timed-out requests\n * are retried with a backoff that starts short and, once it grows to the\n * interval, defers to the regular ticks. Nothing is published until a request\n * succeeds.\n */\nexport class LitestreamSyncRequester {\n  readonly #lc: LogContext;\n  readonly #state = new RunningState('litestream-sync-requester');\n  readonly #config: SyncRequesterConfig;\n  readonly #litestream: SyncController;\n  readonly #readLocalWatermark: Statement;\n  readonly #stream: Subscription<BackedUpWatermark>;\n\n  #timer: NodeJS.Timeout | undefined;\n  #retryTimer: NodeJS.Timeout | undefined;\n  #retryDelayMs = MIN_RETRY_DELAY_MS;\n  #inFlight: Promise<void> | undefined;\n  #backedUp: string | undefined;\n  #lastErrorLogMs = 0;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    config: SyncRequesterConfig,\n    litestream: SyncController = new LitestreamController(lc, replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-sync-requester');\n    this.#config = config;\n    this.#litestream = litestream;\n\n    const db = new Database(this.#lc, replicaFile, {readonly: true});\n    this.#readLocalWatermark = db.prepare(\n      `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n    );\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#timer);\n        clearTimeout(this.#retryTimer);\n        this.#state.stop(this.#lc);\n        this.#litestream.close();\n        db.close();\n      },\n    });\n  }\n\n  /**\n   * Starts requesting syncs and pushes backed-up watermarks to the returned\n   * stream. Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#timer === undefined, `Already called start()`);\n    this.#lc.info?.(\n      `requesting litestream syncs every ${this.#config.intervalMs} ms`,\n    );\n    this.#timer = setInterval(this.tick, this.#config.intervalMs);\n    void this.tick();\n    return this.#stream;\n  }\n\n  /**\n   * Requests a sync unless a request is already outstanding. Resolves when\n   * this tick's request (or the outstanding one) completes.\n   * Exported for testing.\n   */\n  readonly tick = (): Promise<void> => {\n    if (!this.#state.shouldRun() || this.#inFlight) {\n      return this.#inFlight ?? Promise.resolve();\n    }\n    let local: LocalWatermark;\n    try {\n      local = v.parse(this.#readLocalWatermark.get(), localWatermarkSchema);\n    } catch (e) {\n      this.#lc.warn?.(`unable to read local watermark`, e);\n      return Promise.resolve();\n    }\n    this.#inFlight = this.#requestSync(local).finally(() => {\n      this.#inFlight = undefined;\n    });\n    return this.#inFlight;\n  };\n\n  async #requestSync(local: LocalWatermark) {\n    // `local` must be read before the request is issued: litestream only\n    // guarantees that transactions committed before the request are synced.\n    try {\n      const result = await this.#litestream.sync(\n        {\n          wait: true,\n          timeoutMs: CLIENT_TIMEOUT_MS,\n          serverTimeoutSeconds: SERVER_TIMEOUT_SECONDS,\n        },\n        this.#state.signal,\n      );\n      this.#retryDelayMs = MIN_RETRY_DELAY_MS;\n      this.#publish(local, Date.now());\n      this.#lc.debug?.(`backed up watermark ${local.stateVersion}`, result);\n    } catch (e) {\n      if (!this.#state.shouldRun()) {\n        return; // shutting down\n      }\n      // Expected while litestream is starting up or busy with a long sync or\n      // upload (e.g. the initial snapshot of a large replica); the request is\n      // retried after a backoff, or on the next tick if that comes first.\n      this.#scheduleRetry();\n      const now = Date.now();\n      if (now - this.#lastErrorLogMs >= ERROR_LOG_INTERVAL_MS) {\n        this.#lastErrorLogMs = now;\n        this.#lc.info?.(\n          `backup of watermark ${local.stateVersion} not yet confirmed. retrying`,\n          e,\n        );\n      }\n    }\n  }\n\n  #scheduleRetry() {\n    clearTimeout(this.#retryTimer);\n    const delayMs = this.#retryDelayMs;\n    if (delayMs >= this.#config.intervalMs) {\n      return; // backed off to the interval; the regular ticks retry\n    }\n    this.#retryDelayMs = delayMs * 2;\n    this.#retryTimer = setTimeout(this.tick, delayMs);\n  }\n\n  #publish({stateVersion, writeTimeMs}: LocalWatermark, backupTimeMs: number) {\n    if (this.#backedUp !== undefined && stateVersion <= this.#backedUp) {\n      return;\n    }\n    this.#backedUp = stateVersion;\n    this.#stream.push({watermark: stateVersion, writeTimeMs, backupTimeMs});\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAM,yBAAyB;;;;;AAM/B,IAAM,oBAAA;AAEN,IAAM,wBAAwB;;;;;;;;;;AAW9B,IAAM,qBAAqB;AAE3B,IAAM,uBAAuBA,eAAE,OAAO;CACpC,cAAcA,eAAE,OAAO;CACvB,aAAaA,eAAE,OAAO;AACxB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;AAuCD,IAAa,0BAAb,MAAqC;CACnC;CACA,SAAkB,IAAI,aAAa,2BAA2B;CAC9D;CACA;CACA;CACA;CAEA;CACA;CACA,gBAAgB;CAChB;CACA;CACA,kBAAkB;CAElB,YACE,IACA,aACA,QACA,aAA6B,IAAI,qBAAqB,IAAI,WAAW,GACrE;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAK,UAAU;EACf,KAAK,cAAc;EAEnB,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,KAAK,sBAAsB,GAAG,QAC5B,gEACF;EACA,KAAK,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAK,MAAM;GACzB,aAAa,KAAK,WAAW;GAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;GACzB,KAAK,YAAY,MAAM;GACvB,GAAG,MAAM;EACX,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAK,WAAW,KAAA,GAAW,wBAAwB;EAC1D,KAAK,IAAI,OACP,qCAAqC,KAAK,QAAQ,WAAW,IAC/D;EACA,KAAK,SAAS,YAAY,KAAK,MAAM,KAAK,QAAQ,UAAU;EAC5D,KAAU,KAAK;EACf,OAAO,KAAK;CACd;;;;;;CAOA,aAAqC;EACnC,IAAI,CAAC,KAAK,OAAO,UAAU,KAAK,KAAK,WACnC,OAAO,KAAK,aAAa,QAAQ,QAAQ;EAE3C,IAAI;EACJ,IAAI;GACF,QAAQC,MAAQ,KAAK,oBAAoB,IAAI,GAAG,oBAAoB;EACtE,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,kCAAkC,CAAC;GACnD,OAAO,QAAQ,QAAQ;EACzB;EACA,KAAK,YAAY,KAAK,aAAa,KAAK,CAAC,CAAC,cAAc;GACtD,KAAK,YAAY,KAAA;EACnB,CAAC;EACD,OAAO,KAAK;CACd;CAEA,MAAM,aAAa,OAAuB;EAGxC,IAAI;GACF,MAAM,SAAS,MAAM,KAAK,YAAY,KACpC;IACE,MAAM;IACN,WAAW;IACX,sBAAsB;GACxB,GACA,KAAK,OAAO,MACd;GACA,KAAK,gBAAgB;GACrB,KAAK,SAAS,OAAO,KAAK,IAAI,CAAC;GAC/B,KAAK,IAAI,QAAQ,uBAAuB,MAAM,gBAAgB,MAAM;EACtE,SAAS,GAAG;GACV,IAAI,CAAC,KAAK,OAAO,UAAU,GACzB;GAKF,KAAK,eAAe;GACpB,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,MAAM,KAAK,mBAAmB,uBAAuB;IACvD,KAAK,kBAAkB;IACvB,KAAK,IAAI,OACP,uBAAuB,MAAM,aAAa,+BAC1C,CACF;GACF;EACF;CACF;CAEA,iBAAiB;EACf,aAAa,KAAK,WAAW;EAC7B,MAAM,UAAU,KAAK;EACrB,IAAI,WAAW,KAAK,QAAQ,YAC1B;EAEF,KAAK,gBAAgB,UAAU;EAC/B,KAAK,cAAc,WAAW,KAAK,MAAM,OAAO;CAClD;CAEA,SAAS,EAAC,cAAc,eAA8B,cAAsB;EAC1E,IAAI,KAAK,cAAc,KAAA,KAAa,gBAAgB,KAAK,WACvD;EAEF,KAAK,YAAY;EACjB,KAAK,QAAQ,KAAK;GAAC,WAAW;GAAc;GAAa;EAAY,CAAC;CACxE;AACF"}
+{"version":3,"file":"litestream-sync-requester.js","names":["v","v.parse"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream-sync-requester.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {LitestreamController} from '../litestream/litestream-controller.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackedUpWatermark, BackupPoller} from './backup-monitor.ts';\n\n/**\n * How long litestream waits for a requested sync (and its upload) before\n * ending the request. A request that times out is simply retried on a later\n * tick: litestream only abandons the *wait*, never the sync or upload itself,\n * so this bounds request lifetime without affecting backup progress.\n */\nconst SERVER_TIMEOUT_SECONDS = 60;\n\n/**\n * Client-side timeout, slightly above the server timeout so that litestream,\n * rather than the client, ends a slow request.\n */\nconst CLIENT_TIMEOUT_MS = (SERVER_TIMEOUT_SECONDS + 5) * 1000;\n\nconst ERROR_LOG_INTERVAL_MS = 60_000;\n\n/**\n * Initial delay before retrying a failed request, doubled on each consecutive\n * failure until it reaches the configured interval, after which the regular\n * ticks alone retry, so that a busy litestream sees no more requests than the\n * interval allows. The first request is typically issued before litestream's\n * control socket is listening; retrying quickly (rather than after a full\n * interval) keeps the initial backup confirmation, which gates readiness, from\n * waiting out an interval.\n */\nconst MIN_RETRY_DELAY_MS = 250;\n\n/**\n * How often the local replica is polled while a requested sync (see\n * {@link LitestreamSyncRequester.requestSync}) waits for the replica to reach\n * the requested watermark. Each poll is a single read of the replica's\n * watermark, which the replica typically reaches within a poll or two.\n */\nexport const EXPEDITED_POLL_INTERVAL_MS = 50;\n\nconst localWatermarkSchema = v.object({\n  stateVersion: v.string(),\n  writeTimeMs: v.number(),\n});\n\ntype LocalWatermark = v.Infer<typeof localWatermarkSchema>;\n\n/** The subset of {@link LitestreamController} used by the requester. */\nexport type SyncController = Pick<LitestreamController, 'sync' | 'close'>;\n\nexport type SyncRequesterConfig = {\n  /**\n   * Interval between sync requests. This paces how often litestream seals\n   * (and uploads) a new LTX file.\n   */\n  intervalMs: number;\n};\n\n/**\n * The LitestreamSyncRequester drives litestream v5 backups and tracks which\n * replica watermark is durably backed up, emitting it as a stream of\n * {@link BackedUpWatermark}s.\n *\n * It periodically reads the replica's current watermark `W` and then asks\n * litestream to sync and wait for the upload (`POST /sync {wait: true}`).\n * litestream satisfies the request with a sync pass that *starts after* the\n * request, and a pass syncs up to the end of the WAL, so a successful response\n * means that every transaction committed before the request, including the\n * one that wrote `W`, has been uploaded to the backup. No part of the backup\n * needs to be read to determine this.\n *\n * Requests continue even when `W` is already backed up, so that changes that\n * don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)\n * are also backed up at this cadence rather than only at litestream's slower\n * backstop interval. A request on an idle replica is a local no-op for\n * litestream: there is nothing to seal or upload, and no backup reads.\n *\n * At most one request is outstanding at a time. Failed or timed-out requests\n * are retried with a backoff that starts short and, once it grows to the\n * interval, defers to the regular ticks. Nothing is published until a request\n * succeeds.\n *\n * A sync through a specific watermark can also be requested out of band (see\n * {@link requestSync}), e.g. when a replica fork waits on its backup.\n */\nexport class LitestreamSyncRequester implements BackupPoller {\n  readonly #lc: LogContext;\n  readonly #state = new RunningState('litestream-sync-requester');\n  readonly #config: SyncRequesterConfig;\n  readonly #litestream: SyncController;\n  readonly #replicaFile: string;\n  readonly #stream: Subscription<BackedUpWatermark>;\n\n  #db: Database | undefined;\n  #readLocalWatermark: Statement | undefined;\n  #timer: NodeJS.Timeout | undefined;\n  #retryTimer: NodeJS.Timeout | undefined;\n  // The target of a requested sync, and when it stops being expedited.\n  #expedited: {target: string; deadlineMs: number} | undefined;\n  #expediteTimer: NodeJS.Timeout | undefined;\n  #retryDelayMs = MIN_RETRY_DELAY_MS;\n  #inFlight: Promise<void> | undefined;\n  #backedUp: string | undefined;\n  #lastErrorLogMs = 0;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    config: SyncRequesterConfig,\n    litestream: SyncController = new LitestreamController(lc, replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-sync-requester');\n    this.#config = config;\n    this.#litestream = litestream;\n    this.#replicaFile = replicaFile;\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#timer);\n        clearTimeout(this.#retryTimer);\n        clearTimeout(this.#expediteTimer);\n        this.#state.stop(this.#lc);\n        this.#litestream.close();\n        this.#db?.close();\n      },\n    });\n  }\n\n  /**\n   * Starts requesting syncs and pushes backed-up watermarks to the returned\n   * stream. Must only be called once.\n   *\n   * The replica is only opened here, so that a requester that is never\n   * started holds no resources.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#timer === undefined, `Already called start()`);\n    this.#db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n    this.#readLocalWatermark = this.#db.prepare(\n      `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n    );\n    this.#lc.info?.(\n      `requesting litestream syncs every ${this.#config.intervalMs} ms`,\n    );\n    this.#timer = setInterval(this.tick, this.#config.intervalMs);\n    void this.tick();\n    return this.#stream;\n  }\n\n  /**\n   * Requests a sync unless a request is already outstanding. Resolves when\n   * this tick's request (or the outstanding one) completes.\n   * Exported for testing.\n   */\n  readonly tick = (): Promise<void> => {\n    if (!this.#state.shouldRun() || this.#inFlight) {\n      return this.#inFlight ?? Promise.resolve();\n    }\n    const local = this.#readLocal();\n    if (!local) {\n      return Promise.resolve();\n    }\n    this.#inFlight = this.#requestSync(local).finally(() => {\n      this.#inFlight = undefined;\n    });\n    return this.#inFlight;\n  };\n\n  #readLocal(): LocalWatermark | undefined {\n    try {\n      return v.parse(\n        must(this.#readLocalWatermark, `start() was not called`).get(),\n        localWatermarkSchema,\n      );\n    } catch (e) {\n      this.#lc.warn?.(`unable to read local watermark`, e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Requests a sync through `watermark` as soon as the local replica has\n   * reached it, rather than at the next regular tick.\n   *\n   * The replica is polled every {@link EXPEDITED_POLL_INTERVAL_MS} until its\n   * watermark reaches the target, after which a sync is requested\n   * immediately (or once the outstanding request, which may predate the\n   * target, completes). A failed request is retried with the usual backoff.\n   * Concurrent requests are coalesced into one, for the latest target.\n   *\n   * If the replica does not reach the target within the configured interval,\n   * the request is dropped, as the regular ticks then cover it anyway.\n   */\n  requestSync(watermark: string) {\n    if (\n      !this.#state.shouldRun() ||\n      (this.#backedUp !== undefined && this.#backedUp >= watermark)\n    ) {\n      return;\n    }\n    const prev = this.#expedited?.target;\n    this.#expedited = {\n      target: prev !== undefined && prev > watermark ? prev : watermark,\n      deadlineMs: Date.now() + this.#config.intervalMs,\n    };\n    this.#scheduleExpedite(0);\n  }\n\n  #scheduleExpedite(delayMs: number) {\n    clearTimeout(this.#expediteTimer);\n    this.#expediteTimer = setTimeout(this.#expedite, delayMs);\n  }\n\n  readonly #expedite = () => {\n    this.#expediteTimer = undefined;\n    const expedited = this.#expedited;\n    if (!expedited || !this.#state.shouldRun()) {\n      return;\n    }\n    const {target, deadlineMs} = expedited;\n    if (this.#backedUp !== undefined && this.#backedUp >= target) {\n      this.#expedited = undefined; // e.g. by a regular tick\n      return;\n    }\n    if (Date.now() >= deadlineMs) {\n      this.#lc.info?.(\n        `replica did not reach ${target} within ${this.#config.intervalMs} ms. ` +\n          `Deferring its backup to the regular syncs.`,\n      );\n      this.#expedited = undefined;\n      return;\n    }\n    if (this.#inFlight) {\n      // The outstanding request may have read the local watermark before the\n      // replica reached the target. Check again once it completes.\n      void this.#inFlight.finally(() => this.#scheduleExpedite(0));\n      return;\n    }\n    const local = this.#readLocal();\n    if (!local || local.stateVersion < target) {\n      this.#scheduleExpedite(EXPEDITED_POLL_INTERVAL_MS);\n      return;\n    }\n    // A single request covers the target. If it fails, the usual backoff\n    // retries it.\n    this.#expedited = undefined;\n    this.#lc.info?.(`requesting sync through ${target}`);\n    void this.tick();\n  };\n\n  async #requestSync(local: LocalWatermark) {\n    // `local` must be read before the request is issued: litestream only\n    // guarantees that transactions committed before the request are synced.\n    try {\n      const result = await this.#litestream.sync(\n        {\n          wait: true,\n          timeoutMs: CLIENT_TIMEOUT_MS,\n          serverTimeoutSeconds: SERVER_TIMEOUT_SECONDS,\n        },\n        this.#state.signal,\n      );\n      this.#retryDelayMs = MIN_RETRY_DELAY_MS;\n      this.#publish(local, Date.now());\n      this.#lc.debug?.(`backed up watermark ${local.stateVersion}`, result);\n    } catch (e) {\n      if (!this.#state.shouldRun()) {\n        return; // shutting down\n      }\n      // Expected while litestream is starting up or busy with a long sync or\n      // upload (e.g. the initial snapshot of a large replica); the request is\n      // retried after a backoff, or on the next tick if that comes first.\n      this.#scheduleRetry();\n      const now = Date.now();\n      if (now - this.#lastErrorLogMs >= ERROR_LOG_INTERVAL_MS) {\n        this.#lastErrorLogMs = now;\n        this.#lc.info?.(\n          `backup of watermark ${local.stateVersion} not yet confirmed. retrying`,\n          e,\n        );\n      }\n    }\n  }\n\n  #scheduleRetry() {\n    clearTimeout(this.#retryTimer);\n    const delayMs = this.#retryDelayMs;\n    if (delayMs >= this.#config.intervalMs) {\n      return; // backed off to the interval; the regular ticks retry\n    }\n    this.#retryDelayMs = delayMs * 2;\n    this.#retryTimer = setTimeout(this.tick, delayMs);\n  }\n\n  #publish({stateVersion, writeTimeMs}: LocalWatermark, backupTimeMs: number) {\n    if (this.#backedUp !== undefined && stateVersion <= this.#backedUp) {\n      return;\n    }\n    this.#backedUp = stateVersion;\n    this.#stream.push({watermark: stateVersion, writeTimeMs, backupTimeMs});\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAiBA,IAAM,yBAAyB;;;;;AAM/B,IAAM,oBAAA;AAEN,IAAM,wBAAwB;;;;;;;;;;AAW9B,IAAM,qBAAqB;AAU3B,IAAM,uBAAuBA,eAAE,OAAO;CACpC,cAAcA,eAAE,OAAO;CACvB,aAAaA,eAAE,OAAO;AACxB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0CD,IAAa,0BAAb,MAA6D;CAC3D;CACA,SAAkB,IAAI,aAAa,2BAA2B;CAC9D;CACA;CACA;CACA;CAEA;CACA;CACA;CACA;CAEA;CACA;CACA,gBAAgB;CAChB;CACA;CACA,kBAAkB;CAElB,YACE,IACA,aACA,QACA,aAA6B,IAAI,qBAAqB,IAAI,WAAW,GACrE;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAK,UAAU;EACf,KAAK,cAAc;EACnB,KAAK,eAAe;EACpB,KAAK,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAK,MAAM;GACzB,aAAa,KAAK,WAAW;GAC7B,aAAa,KAAK,cAAc;GAChC,KAAK,OAAO,KAAK,KAAK,GAAG;GACzB,KAAK,YAAY,MAAM;GACvB,KAAK,KAAK,MAAM;EAClB,EACF,CAAC;CACH;;;;;;;;CASA,QAAmC;EACjC,OAAO,KAAK,WAAW,KAAA,GAAW,wBAAwB;EAC1D,KAAK,MAAM,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;EACrE,KAAK,sBAAsB,KAAK,IAAI,QAClC,gEACF;EACA,KAAK,IAAI,OACP,qCAAqC,KAAK,QAAQ,WAAW,IAC/D;EACA,KAAK,SAAS,YAAY,KAAK,MAAM,KAAK,QAAQ,UAAU;EAC5D,KAAU,KAAK;EACf,OAAO,KAAK;CACd;;;;;;CAOA,aAAqC;EACnC,IAAI,CAAC,KAAK,OAAO,UAAU,KAAK,KAAK,WACnC,OAAO,KAAK,aAAa,QAAQ,QAAQ;EAE3C,MAAM,QAAQ,KAAK,WAAW;EAC9B,IAAI,CAAC,OACH,OAAO,QAAQ,QAAQ;EAEzB,KAAK,YAAY,KAAK,aAAa,KAAK,CAAC,CAAC,cAAc;GACtD,KAAK,YAAY,KAAA;EACnB,CAAC;EACD,OAAO,KAAK;CACd;CAEA,aAAyC;EACvC,IAAI;GACF,OAAOC,MACL,KAAK,KAAK,qBAAqB,wBAAwB,CAAC,CAAC,IAAI,GAC7D,oBACF;EACF,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,kCAAkC,CAAC;GACnD;EACF;CACF;;;;;;;;;;;;;;CAeA,YAAY,WAAmB;EAC7B,IACE,CAAC,KAAK,OAAO,UAAU,KACtB,KAAK,cAAc,KAAA,KAAa,KAAK,aAAa,WAEnD;EAEF,MAAM,OAAO,KAAK,YAAY;EAC9B,KAAK,aAAa;GAChB,QAAQ,SAAS,KAAA,KAAa,OAAO,YAAY,OAAO;GACxD,YAAY,KAAK,IAAI,IAAI,KAAK,QAAQ;EACxC;EACA,KAAK,kBAAkB,CAAC;CAC1B;CAEA,kBAAkB,SAAiB;EACjC,aAAa,KAAK,cAAc;EAChC,KAAK,iBAAiB,WAAW,KAAK,WAAW,OAAO;CAC1D;CAEA,kBAA2B;EACzB,KAAK,iBAAiB,KAAA;EACtB,MAAM,YAAY,KAAK;EACvB,IAAI,CAAC,aAAa,CAAC,KAAK,OAAO,UAAU,GACvC;EAEF,MAAM,EAAC,QAAQ,eAAc;EAC7B,IAAI,KAAK,cAAc,KAAA,KAAa,KAAK,aAAa,QAAQ;GAC5D,KAAK,aAAa,KAAA;GAClB;EACF;EACA,IAAI,KAAK,IAAI,KAAK,YAAY;GAC5B,KAAK,IAAI,OACP,yBAAyB,OAAO,UAAU,KAAK,QAAQ,WAAW,gDAEpE;GACA,KAAK,aAAa,KAAA;GAClB;EACF;EACA,IAAI,KAAK,WAAW;GAGlB,KAAU,UAAU,cAAc,KAAK,kBAAkB,CAAC,CAAC;GAC3D;EACF;EACA,MAAM,QAAQ,KAAK,WAAW;EAC9B,IAAI,CAAC,SAAS,MAAM,eAAe,QAAQ;GACzC,KAAK,kBAAA,EAA4C;GACjD;EACF;EAGA,KAAK,aAAa,KAAA;EAClB,KAAK,IAAI,OAAO,2BAA2B,QAAQ;EACnD,KAAU,KAAK;CACjB;CAEA,MAAM,aAAa,OAAuB;EAGxC,IAAI;GACF,MAAM,SAAS,MAAM,KAAK,YAAY,KACpC;IACE,MAAM;IACN,WAAW;IACX,sBAAsB;GACxB,GACA,KAAK,OAAO,MACd;GACA,KAAK,gBAAgB;GACrB,KAAK,SAAS,OAAO,KAAK,IAAI,CAAC;GAC/B,KAAK,IAAI,QAAQ,uBAAuB,MAAM,gBAAgB,MAAM;EACtE,SAAS,GAAG;GACV,IAAI,CAAC,KAAK,OAAO,UAAU,GACzB;GAKF,KAAK,eAAe;GACpB,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,MAAM,KAAK,mBAAmB,uBAAuB;IACvD,KAAK,kBAAkB;IACvB,KAAK,IAAI,OACP,uBAAuB,MAAM,aAAa,+BAC1C,CACF;GACF;EACF;CACF;CAEA,iBAAiB;EACf,aAAa,KAAK,WAAW;EAC7B,MAAM,UAAU,KAAK;EACrB,IAAI,WAAW,KAAK,QAAQ,YAC1B;EAEF,KAAK,gBAAgB,UAAU;EAC/B,KAAK,cAAc,WAAW,KAAK,MAAM,OAAO;CAClD;CAEA,SAAS,EAAC,cAAc,eAA8B,cAAsB;EAC1E,IAAI,KAAK,cAAc,KAAA,KAAa,gBAAgB,KAAK,WACvD;EAEF,KAAK,YAAY;EACjB,KAAK,QAAQ,KAAK;GAAC,WAAW;GAAc;GAAa;EAAY,CAAC;CACxE;AACF"}

package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map

@@ -1 +1 @@
-{"version":3,"file":"litestream3-prometheus-poller.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamBackupVerificationDuration,\n  litestreamMonitorMetricAttrs,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nexport const CHECK_INTERVAL_MS = 10_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\nconst GiB = 1024 ** 3;\n\n/**\n * How long the BackupMonitor waits for the *first* restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n *\n * On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n */\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/**\n * How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n */\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\n/**\n * The Litestream3PrometheusPoller polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Because the watermarks reported by litestream metrics reflect what\n * litestream *believes* has been backed up (which has been observed to\n * diverge from reality when uploads silently fail), the cleanup watermark is\n * only published after verifying it against the actual backup state in the\n * replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class Litestream3PrometheusPoller {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #state = new RunningState('litestream3-prometheus-poller');\n\n  readonly #stream: Subscription<BackedUpWatermark>;\n  readonly #watermarks = new Map<string, Date>();\n\n  readonly #verifyBackupState: BackupStateVerifier;\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup state ' +\n      '(as listed from the replica destination) could not be verified, or ' +\n      'is older than the backup progress claimed by litestream metrics. ' +\n      'A steadily increasing value indicates a wedged or failing backup. ' +\n      'The \"backup-wedged\" reason is emitted once just before the process ' +\n      'shuts itself down due to a persistently stale backup.',\n  });\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  // The time of the most recent object actually verified in the backup\n  // destination, or `null` if none has ever been verified. Only assigned from a\n  // successful verifyBackupState() and never cleared, so a non-null value also\n  // means a restorable backup has been confirmed to exist (which is what gates\n  // the snapshot `status` signal and satisfies the initial-backup deadline).\n  #lastVerifiedUploadTime: Date | null = null;\n  // Epoch ms at which the actual backup state was first observed to be\n  // continuously behind the watermark litestream claims to have backed up,\n  // or `null` while the backup is keeping up. Reset whenever a purge is\n  // confirmed against a real upload.\n  #backupStaleSince: number | null = null;\n  // Epoch ms at which run() started, used to bound how long to wait for the\n  // first restorable backup to appear. `null` until run() is called.\n  #runStartTime: number | null = null;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    verifyBackupState: BackupStateVerifier,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream3-prometheus-poller');\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#verifyBackupState = verifyBackupState;\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#checkMetricsTimer);\n        this.#state.stop(this.#lc);\n      },\n    });\n  }\n\n  /**\n   * Starts polling for watermarks and pushes new ones to returned stream.\n   * Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#runStartTime === null, `Already called start()`);\n    this.#runStartTime = Date.now();\n    this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);\n    this.#checkMetricsTimer = setInterval(\n      this.checkVerifyAndPublishWatermarks,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#stream;\n  }\n\n  /**\n   * Verifies whether a restorable backup actually exists in the destination\n   * (as opposed to what litestream metrics merely claim). On success the\n   * verified upload time is cached — seeding the cleanup-verification fast path\n   * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n   * Returns false if no backup is listable yet (e.g. the initial snapshot is\n   * still uploading).\n   */\n  async #confirmRestorableBackup(): Promise<boolean> {\n    try {\n      this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      return true;\n    } catch (e) {\n      this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n      return false;\n    }\n  }\n\n  // Exported for testing\n  readonly checkVerifyAndPublishWatermarks = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      await this.#publishVerifiedWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n    try {\n      await this.#checkInitialBackupDeadline();\n    } catch (e) {\n      this.#lc.warn?.(`error checking initial backup deadline`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n          if (watermark) {\n            try {\n              versionFromLexi(watermark); // Sanity check: Ensure that it is valid.\n              yield {watermark, time, name};\n            } catch (e) {\n              this.#lc.warn?.(\n                `ignoring invalid watermark ${watermark}`,\n                {metric},\n                e,\n              );\n            }\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  async #publishVerifiedWatermarks() {\n    const maxBackup = this.#maxBackupBefore(Date.now());\n    if (!maxBackup) {\n      return;\n    }\n    const {watermark: maxWatermark, backupTime: claimedTime} = maxBackup;\n    // Purge guard: the watermarks (and their backup times) come from\n    // litestream metrics, which are exported when litestream *believes*\n    // an upload succeeded, and have been observed to advance even when\n    // nothing was actually written to the backup destination. Purging the\n    // change-log based on a falsely advancing watermark permanently breaks\n    // the ability to restore + catch up. Before advancing the cleanup\n    // watermark, verify it against the actual backup state: a claimed\n    // backup time is only trusted if an object was actually uploaded to\n    // the replica destination at (or after) that time, modulo clock skew.\n    if (!this.#confirmedDurable(claimedTime)) {\n      try {\n        this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      } catch (e) {\n        this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n        // Skipping the purge is safe: the change-log just grows.\n        this.#lc.warn?.(\n          `unable to verify backup state. skipping change-log cleanup ` +\n            `up to watermark ${maxWatermark} ` +\n            `(claimed backup time ${claimedTime.toISOString()})`,\n          e,\n        );\n        return;\n      }\n    }\n    // Watermarks whose backup time isn't yet confirmed durable remain in the\n    // map and are re-evaluated at the next check.\n    const lastUpload = must(this.#lastVerifiedUploadTime);\n    const verifiedBackup = this.#maxBackupBefore(\n      lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n    );\n    if (!verifiedBackup) {\n      this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n      const now = Date.now();\n      if (this.#backupStaleSince === null) {\n        this.#backupStaleSince = now;\n      }\n      const staleForMs = now - this.#backupStaleSince;\n      this.#lc.warn?.(\n        `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n          `litestream claims it was backed up at ` +\n          `${claimedTime.toISOString()}, but the last object actually ` +\n          `uploaded to ${this.#backupURL} was at ` +\n          `${lastUpload.toISOString()}. ` +\n          `The backup has been stale for ${staleForMs} ms.`,\n      );\n      if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n        this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n      }\n      return;\n    }\n    // The cleanup watermark advanced past a real upload, so the backup is\n    // keeping up: clear the staleness clock.\n    this.#backupStaleSince = null;\n    const {watermark: verifiedWatermark, backupTime} = verifiedBackup;\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= verifiedWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#stream.push({\n      watermark: verifiedWatermark,\n      backupTimeMs: backupTime.getTime(),\n    });\n    this.#lastWatermark = verifiedWatermark;\n  }\n\n  async #verifyBackupStateTimed(): Promise<Date> {\n    const start = performance.now();\n    let result: 'success' | 'error' = 'error';\n    try {\n      const ret = await this.#verifyBackupState();\n      result = 'success';\n      return ret;\n    } finally {\n      litestreamBackupVerificationDuration().recordMs(\n        performance.now() - start,\n        {\n          ...litestreamMonitorMetricAttrs(\n            this.#backupURL,\n            'legacy',\n            'replication_manager',\n          ),\n          result,\n        },\n      );\n    }\n  }\n\n  /**\n   * Called when the backup has been continuously stale for longer than\n   * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n   * backup, so the process is shut down by rejecting the {@link run()}\n   * promise. The exit is non-zero and logged at ERROR for alerting; on\n   * restart the monitor re-verifies, so a still-wedged backup keeps the\n   * process down until the destination is fixed.\n   */\n  #shutDownOnWedgedBackup(\n    staleForMs: number,\n    claimedTime: Date,\n    lastUpload: Date,\n  ) {\n    this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n    const err = new UnrecoverableError(\n      `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n        `at ${claimedTime.toISOString()}, but the last object actually ` +\n        `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n        `not advanced for ${staleForMs} ms (grace period ` +\n        `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n        `the backup; investigate the backup destination.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * On a fresh stack, fails loudly if no restorable backup has appeared within\n   * a generous, replica-size-scaled grace period (see\n   * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n   * onto the replication-manager — the producer responsible for creating the\n   * backup — instead of leaving view-syncers to wait (and the platform to keep\n   * restarting them) while the real fault is here. Does nothing once a\n   * restorable backup has been confirmed.\n   */\n  async #checkInitialBackupDeadline() {\n    if (this.#lastVerifiedUploadTime !== null || this.#runStartTime === null) {\n      return;\n    }\n    const elapsed = Date.now() - this.#runStartTime;\n    if (elapsed < this.#initialBackupGraceMs()) {\n      return;\n    }\n    // The deadline has elapsed without a confirmed backup. Do one definitive\n    // check against the destination before giving up, in case a backup landed\n    // but was never exercised by a reservation or a cleanup verification.\n    if (await this.#confirmRestorableBackup()) {\n      return;\n    }\n    this.#shutDownOnMissingInitialBackup(elapsed);\n  }\n\n  /**\n   * The grace period for the first restorable backup, scaled by replica size to\n   * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n   * the minimum (one unit) if the replica file size cannot be determined.\n   */\n  #initialBackupGraceMs(): number {\n    let bytes = 0;\n    try {\n      bytes = statSync(this.#replicaFile).size;\n    } catch {\n      // Replica file not present/readable yet; fall back to the minimum grace.\n    }\n    const units = Math.max(\n      1,\n      Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n    );\n    return units * INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n  }\n\n  /**\n   * Called when no restorable backup has appeared within\n   * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n   * view-syncer can restore — and the fault is the backup pipeline, so the\n   * process exits non-zero with a logged error for alerting, mirroring\n   * {@link #shutDownOnWedgedBackup}.\n   */\n  #shutDownOnMissingInitialBackup(elapsedMs: number) {\n    const err = new UnrecoverableError(\n      `no restorable backup has appeared at ${this.#backupURL} within ` +\n        `${elapsedMs} ms of startup (grace period ` +\n        `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n        `appears to be broken; shutting down so that the replication-manager ` +\n        `is flagged as the cause rather than the view-syncers waiting on it. ` +\n        `Investigate litestream replication to ${this.#backupURL}.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * Returns the newest watermark whose backup time is at or before `cutoff`\n   * (epoch ms), or `''` if there is none.\n   */\n  #maxBackupBefore(\n    cutoff: number,\n  ): {watermark: string; backupTime: Date} | null {\n    let max = '';\n    let maxBackupTime: Date | undefined;\n    for (const [watermark, backupTime] of this.#watermarks) {\n      if (backupTime.getTime() <= cutoff && watermark > max) {\n        max = watermark;\n        maxBackupTime = backupTime;\n      }\n    }\n    if (!maxBackupTime) {\n      return null;\n    }\n    return {watermark: max, backupTime: maxBackupTime};\n  }\n\n  /**\n   * Returns `true` if the actual backup state, as last verified against the\n   * backup destination, confirms that data claimed to be backed up at\n   * `claimedTime` is durable (i.e. an object was actually uploaded at or\n   * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n   * clock skew).\n   */\n  #confirmedDurable(claimedTime: Date): boolean {\n    const lastUpload = this.#lastVerifiedUploadTime;\n    return (\n      lastUpload !== null &&\n      claimedTime.getTime() <=\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n    );\n  }\n}\n"],"mappings":";;;;;;;;;;AAgBA,IAAa,oBAAoB;;;;;AAMjC,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B;AAExC,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC;AAChD,IAAM,kCAAkC,KAAK;;;;;;;;;;;;;;;;;AAmC7C,IAAa,8BAAb,MAAyC;CACvC;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,+BAA+B;CAElE;CACA,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAED,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CAEA,YACE,IACA,aACA,WACA,iBACA,mBACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,+BAA+B;EACtE,KAAK,eAAe;EACpB,KAAK,aAAa;EAClB,KAAK,mBAAmB;EACxB,KAAK,qBAAqB;EAC1B,KAAK,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAK,kBAAkB;GACrC,KAAK,OAAO,KAAK,KAAK,GAAG;EAC3B,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAK,kBAAkB,MAAM,wBAAwB;EAC5D,KAAK,gBAAgB,KAAK,IAAI;EAC9B,KAAK,IAAI,OAAO,yBAAyB,KAAK,kBAAkB;EAChE,KAAK,qBAAqB,YACxB,KAAK,iCACL,iBACF;EACA,OAAO,KAAK;CACd;;;;;;;;;CAUA,MAAM,2BAA6C;EACjD,IAAI;GACF,KAAK,0BAA0B,MAAM,KAAK,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,gCAAgC,KAAK,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAGA,kCAA2C,YAAY;EACrD,IAAI;GACF,MAAM,KAAK,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,8BAA8B,KAAK,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAK,2BAA2B;EACxC,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAK,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAO,mBAIJ;EACD,MAAM,kBAAkB,KAAK;EAC7B,MAAM,SAAS,KAAK,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAK,IAAI,OAAO,8BAA8B,KAAK,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAK,IAAI,OACP,8BAA8B,KAAK,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GACrD,IAAI,WACF,IAAI;IACF,gBAAgB,SAAS;IACzB,MAAM;KAAC;KAAW;KAAM;IAAI;GAC9B,SAAS,GAAG;IACV,KAAK,IAAI,OACP,8BAA8B,aAC9B,EAAC,OAAM,GACP,CACF;GACF;EAEJ;CAGN;CAEA,MAAM,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAK,iBAAiB,GAChE,IAAI,YAAY,KAAK,kBAAkB,CAAC,KAAK,YAAY,IAAI,SAAS,GAAG;GACvE,KAAK,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAK,YAAY,IAAI,WAAW,IAAI;GACpC,KAAK,oBAAoB;EAC3B;EAEF,OAAO,KAAK;CACd;CAEA,MAAM,6BAA6B;EACjC,MAAM,YAAY,KAAK,iBAAiB,KAAK,IAAI,CAAC;EAClD,IAAI,CAAC,WACH;EAEF,MAAM,EAAC,WAAW,cAAc,YAAY,gBAAe;EAU3D,IAAI,CAAC,KAAK,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAK,0BAA0B,MAAM,KAAK,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAK,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAK,uBAAuB;EACpD,MAAM,iBAAiB,KAAK,iBAC1B,WAAW,QAAQ,IAAI,4BACzB;EACA,IAAI,CAAC,gBAAgB;GACnB,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAK,sBAAsB,MAC7B,KAAK,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAK;GAC9B,KAAK,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAK,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAK,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAK,oBAAoB;EACzB,MAAM,EAAC,WAAW,mBAAmB,eAAc;EACnD,KAAK,MAAM,aAAa,KAAK,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAK,YAAY,OAAO,SAAS;EAGrC,KAAK,QAAQ,KAAK;GAChB,WAAW;GACX,cAAc,WAAW,QAAQ;EACnC,CAAC;EACD,KAAK,iBAAiB;CACxB;CAEA,MAAM,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAK,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,CAAC,CAAC,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAK,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAK,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAK,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAK,QAAQ,KAAK,GAAG;CACvB;;;;;;;;;;CAWA,MAAM,8BAA8B;EAClC,IAAI,KAAK,4BAA4B,QAAQ,KAAK,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAK;EAClC,IAAI,UAAU,KAAK,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAK,yBAAyB,GACtC;EAEF,KAAK,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAK,YAAY,CAAC,CAAC;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAK,WAAW,UACnD,UAAU,+BACV,KAAK,sBAAsB,EAAE,kNAGS,KAAK,WAAW,EAC7D;EACA,KAAK,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAK,QAAQ,KAAK,GAAG;CACvB;;;;;CAMA,iBACE,QAC8C;EAC9C,IAAI,MAAM;EACV,IAAI;EACJ,KAAK,MAAM,CAAC,WAAW,eAAe,KAAK,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAAK;GACrD,MAAM;GACN,gBAAgB;EAClB;EAEF,IAAI,CAAC,eACH,OAAO;EAET,OAAO;GAAC,WAAW;GAAK,YAAY;EAAa;CACnD;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAK;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;AACF"}
+{"version":3,"file":"litestream3-prometheus-poller.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamBackupVerificationDuration,\n  litestreamMonitorMetricAttrs,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackedUpWatermark, BackupPoller} from './backup-monitor.ts';\n\nexport const CHECK_INTERVAL_MS = 10_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\nconst GiB = 1024 ** 3;\n\n/**\n * How long the BackupMonitor waits for the *first* restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n *\n * On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n */\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/**\n * How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n */\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\n/**\n * The Litestream3PrometheusPoller polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Because the watermarks reported by litestream metrics reflect what\n * litestream *believes* has been backed up (which has been observed to\n * diverge from reality when uploads silently fail), the cleanup watermark is\n * only published after verifying it against the actual backup state in the\n * replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class Litestream3PrometheusPoller implements BackupPoller {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #state = new RunningState('litestream3-prometheus-poller');\n\n  readonly #stream: Subscription<BackedUpWatermark>;\n  readonly #watermarks = new Map<string, Date>();\n\n  readonly #verifyBackupState: BackupStateVerifier;\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup state ' +\n      '(as listed from the replica destination) could not be verified, or ' +\n      'is older than the backup progress claimed by litestream metrics. ' +\n      'A steadily increasing value indicates a wedged or failing backup. ' +\n      'The \"backup-wedged\" reason is emitted once just before the process ' +\n      'shuts itself down due to a persistently stale backup.',\n  });\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  // The time of the most recent object actually verified in the backup\n  // destination, or `null` if none has ever been verified. Only assigned from a\n  // successful verifyBackupState() and never cleared, so a non-null value also\n  // means a restorable backup has been confirmed to exist (which is what gates\n  // the snapshot `status` signal and satisfies the initial-backup deadline).\n  #lastVerifiedUploadTime: Date | null = null;\n  // Epoch ms at which the actual backup state was first observed to be\n  // continuously behind the watermark litestream claims to have backed up,\n  // or `null` while the backup is keeping up. Reset whenever a purge is\n  // confirmed against a real upload.\n  #backupStaleSince: number | null = null;\n  // Epoch ms at which run() started, used to bound how long to wait for the\n  // first restorable backup to appear. `null` until run() is called.\n  #runStartTime: number | null = null;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    verifyBackupState: BackupStateVerifier,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream3-prometheus-poller');\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#verifyBackupState = verifyBackupState;\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#checkMetricsTimer);\n        this.#state.stop(this.#lc);\n      },\n    });\n  }\n\n  /**\n   * Starts polling for watermarks and pushes new ones to returned stream.\n   * Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#runStartTime === null, `Already called start()`);\n    this.#runStartTime = Date.now();\n    this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);\n    this.#checkMetricsTimer = setInterval(\n      this.checkVerifyAndPublishWatermarks,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#stream;\n  }\n\n  /**\n   * Verifies whether a restorable backup actually exists in the destination\n   * (as opposed to what litestream metrics merely claim). On success the\n   * verified upload time is cached — seeding the cleanup-verification fast path\n   * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n   * Returns false if no backup is listable yet (e.g. the initial snapshot is\n   * still uploading).\n   */\n  async #confirmRestorableBackup(): Promise<boolean> {\n    try {\n      this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      return true;\n    } catch (e) {\n      this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n      return false;\n    }\n  }\n\n  // Exported for testing\n  readonly checkVerifyAndPublishWatermarks = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      await this.#publishVerifiedWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n    try {\n      await this.#checkInitialBackupDeadline();\n    } catch (e) {\n      this.#lc.warn?.(`error checking initial backup deadline`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n          if (watermark) {\n            try {\n              versionFromLexi(watermark); // Sanity check: Ensure that it is valid.\n              yield {watermark, time, name};\n            } catch (e) {\n              this.#lc.warn?.(\n                `ignoring invalid watermark ${watermark}`,\n                {metric},\n                e,\n              );\n            }\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  async #publishVerifiedWatermarks() {\n    const maxBackup = this.#maxBackupBefore(Date.now());\n    if (!maxBackup) {\n      return;\n    }\n    const {watermark: maxWatermark, backupTime: claimedTime} = maxBackup;\n    // Purge guard: the watermarks (and their backup times) come from\n    // litestream metrics, which are exported when litestream *believes*\n    // an upload succeeded, and have been observed to advance even when\n    // nothing was actually written to the backup destination. Purging the\n    // change-log based on a falsely advancing watermark permanently breaks\n    // the ability to restore + catch up. Before advancing the cleanup\n    // watermark, verify it against the actual backup state: a claimed\n    // backup time is only trusted if an object was actually uploaded to\n    // the replica destination at (or after) that time, modulo clock skew.\n    if (!this.#confirmedDurable(claimedTime)) {\n      try {\n        this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      } catch (e) {\n        this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n        // Skipping the purge is safe: the change-log just grows.\n        this.#lc.warn?.(\n          `unable to verify backup state. skipping change-log cleanup ` +\n            `up to watermark ${maxWatermark} ` +\n            `(claimed backup time ${claimedTime.toISOString()})`,\n          e,\n        );\n        return;\n      }\n    }\n    // Watermarks whose backup time isn't yet confirmed durable remain in the\n    // map and are re-evaluated at the next check.\n    const lastUpload = must(this.#lastVerifiedUploadTime);\n    const verifiedBackup = this.#maxBackupBefore(\n      lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n    );\n    if (!verifiedBackup) {\n      this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n      const now = Date.now();\n      if (this.#backupStaleSince === null) {\n        this.#backupStaleSince = now;\n      }\n      const staleForMs = now - this.#backupStaleSince;\n      this.#lc.warn?.(\n        `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n          `litestream claims it was backed up at ` +\n          `${claimedTime.toISOString()}, but the last object actually ` +\n          `uploaded to ${this.#backupURL} was at ` +\n          `${lastUpload.toISOString()}. ` +\n          `The backup has been stale for ${staleForMs} ms.`,\n      );\n      if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n        this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n      }\n      return;\n    }\n    // The cleanup watermark advanced past a real upload, so the backup is\n    // keeping up: clear the staleness clock.\n    this.#backupStaleSince = null;\n    const {watermark: verifiedWatermark, backupTime} = verifiedBackup;\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= verifiedWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#stream.push({\n      watermark: verifiedWatermark,\n      backupTimeMs: backupTime.getTime(),\n    });\n    this.#lastWatermark = verifiedWatermark;\n  }\n\n  async #verifyBackupStateTimed(): Promise<Date> {\n    const start = performance.now();\n    let result: 'success' | 'error' = 'error';\n    try {\n      const ret = await this.#verifyBackupState();\n      result = 'success';\n      return ret;\n    } finally {\n      litestreamBackupVerificationDuration().recordMs(\n        performance.now() - start,\n        {\n          ...litestreamMonitorMetricAttrs(\n            this.#backupURL,\n            'legacy',\n            'replication_manager',\n          ),\n          result,\n        },\n      );\n    }\n  }\n\n  /**\n   * Called when the backup has been continuously stale for longer than\n   * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n   * backup, so the process is shut down by rejecting the {@link run()}\n   * promise. The exit is non-zero and logged at ERROR for alerting; on\n   * restart the monitor re-verifies, so a still-wedged backup keeps the\n   * process down until the destination is fixed.\n   */\n  #shutDownOnWedgedBackup(\n    staleForMs: number,\n    claimedTime: Date,\n    lastUpload: Date,\n  ) {\n    this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n    const err = new UnrecoverableError(\n      `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n        `at ${claimedTime.toISOString()}, but the last object actually ` +\n        `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n        `not advanced for ${staleForMs} ms (grace period ` +\n        `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n        `the backup; investigate the backup destination.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * On a fresh stack, fails loudly if no restorable backup has appeared within\n   * a generous, replica-size-scaled grace period (see\n   * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n   * onto the replication-manager — the producer responsible for creating the\n   * backup — instead of leaving view-syncers to wait (and the platform to keep\n   * restarting them) while the real fault is here. Does nothing once a\n   * restorable backup has been confirmed.\n   */\n  async #checkInitialBackupDeadline() {\n    if (this.#lastVerifiedUploadTime !== null || this.#runStartTime === null) {\n      return;\n    }\n    const elapsed = Date.now() - this.#runStartTime;\n    if (elapsed < this.#initialBackupGraceMs()) {\n      return;\n    }\n    // The deadline has elapsed without a confirmed backup. Do one definitive\n    // check against the destination before giving up, in case a backup landed\n    // but was never exercised by a reservation or a cleanup verification.\n    if (await this.#confirmRestorableBackup()) {\n      return;\n    }\n    this.#shutDownOnMissingInitialBackup(elapsed);\n  }\n\n  /**\n   * The grace period for the first restorable backup, scaled by replica size to\n   * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n   * the minimum (one unit) if the replica file size cannot be determined.\n   */\n  #initialBackupGraceMs(): number {\n    let bytes = 0;\n    try {\n      bytes = statSync(this.#replicaFile).size;\n    } catch {\n      // Replica file not present/readable yet; fall back to the minimum grace.\n    }\n    const units = Math.max(\n      1,\n      Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n    );\n    return units * INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n  }\n\n  /**\n   * Called when no restorable backup has appeared within\n   * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n   * view-syncer can restore — and the fault is the backup pipeline, so the\n   * process exits non-zero with a logged error for alerting, mirroring\n   * {@link #shutDownOnWedgedBackup}.\n   */\n  #shutDownOnMissingInitialBackup(elapsedMs: number) {\n    const err = new UnrecoverableError(\n      `no restorable backup has appeared at ${this.#backupURL} within ` +\n        `${elapsedMs} ms of startup (grace period ` +\n        `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n        `appears to be broken; shutting down so that the replication-manager ` +\n        `is flagged as the cause rather than the view-syncers waiting on it. ` +\n        `Investigate litestream replication to ${this.#backupURL}.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * Returns the newest watermark whose backup time is at or before `cutoff`\n   * (epoch ms), or `''` if there is none.\n   */\n  #maxBackupBefore(\n    cutoff: number,\n  ): {watermark: string; backupTime: Date} | null {\n    let max = '';\n    let maxBackupTime: Date | undefined;\n    for (const [watermark, backupTime] of this.#watermarks) {\n      if (backupTime.getTime() <= cutoff && watermark > max) {\n        max = watermark;\n        maxBackupTime = backupTime;\n      }\n    }\n    if (!maxBackupTime) {\n      return null;\n    }\n    return {watermark: max, backupTime: maxBackupTime};\n  }\n\n  /**\n   * Returns `true` if the actual backup state, as last verified against the\n   * backup destination, confirms that data claimed to be backed up at\n   * `claimedTime` is durable (i.e. an object was actually uploaded at or\n   * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n   * clock skew).\n   */\n  #confirmedDurable(claimedTime: Date): boolean {\n    const lastUpload = this.#lastVerifiedUploadTime;\n    return (\n      lastUpload !== null &&\n      claimedTime.getTime() <=\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n    );\n  }\n}\n"],"mappings":";;;;;;;;;;AAgBA,IAAa,oBAAoB;;;;;AAMjC,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B;AAExC,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC;AAChD,IAAM,kCAAkC,KAAK;;;;;;;;;;;;;;;;;AAmC7C,IAAa,8BAAb,MAAiE;CAC/D;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,+BAA+B;CAElE;CACA,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAED,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CAEA,YACE,IACA,aACA,WACA,iBACA,mBACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,+BAA+B;EACtE,KAAK,eAAe;EACpB,KAAK,aAAa;EAClB,KAAK,mBAAmB;EACxB,KAAK,qBAAqB;EAC1B,KAAK,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAK,kBAAkB;GACrC,KAAK,OAAO,KAAK,KAAK,GAAG;EAC3B,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAK,kBAAkB,MAAM,wBAAwB;EAC5D,KAAK,gBAAgB,KAAK,IAAI;EAC9B,KAAK,IAAI,OAAO,yBAAyB,KAAK,kBAAkB;EAChE,KAAK,qBAAqB,YACxB,KAAK,iCACL,iBACF;EACA,OAAO,KAAK;CACd;;;;;;;;;CAUA,MAAM,2BAA6C;EACjD,IAAI;GACF,KAAK,0BAA0B,MAAM,KAAK,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,gCAAgC,KAAK,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAGA,kCAA2C,YAAY;EACrD,IAAI;GACF,MAAM,KAAK,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,8BAA8B,KAAK,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAK,2BAA2B;EACxC,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAK,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAO,mBAIJ;EACD,MAAM,kBAAkB,KAAK;EAC7B,MAAM,SAAS,KAAK,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAK,IAAI,OAAO,8BAA8B,KAAK,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAK,IAAI,OACP,8BAA8B,KAAK,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GACrD,IAAI,WACF,IAAI;IACF,gBAAgB,SAAS;IACzB,MAAM;KAAC;KAAW;KAAM;IAAI;GAC9B,SAAS,GAAG;IACV,KAAK,IAAI,OACP,8BAA8B,aAC9B,EAAC,OAAM,GACP,CACF;GACF;EAEJ;CAGN;CAEA,MAAM,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAK,iBAAiB,GAChE,IAAI,YAAY,KAAK,kBAAkB,CAAC,KAAK,YAAY,IAAI,SAAS,GAAG;GACvE,KAAK,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAK,YAAY,IAAI,WAAW,IAAI;GACpC,KAAK,oBAAoB;EAC3B;EAEF,OAAO,KAAK;CACd;CAEA,MAAM,6BAA6B;EACjC,MAAM,YAAY,KAAK,iBAAiB,KAAK,IAAI,CAAC;EAClD,IAAI,CAAC,WACH;EAEF,MAAM,EAAC,WAAW,cAAc,YAAY,gBAAe;EAU3D,IAAI,CAAC,KAAK,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAK,0BAA0B,MAAM,KAAK,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAK,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAK,uBAAuB;EACpD,MAAM,iBAAiB,KAAK,iBAC1B,WAAW,QAAQ,IAAI,4BACzB;EACA,IAAI,CAAC,gBAAgB;GACnB,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAK,sBAAsB,MAC7B,KAAK,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAK;GAC9B,KAAK,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAK,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAK,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAK,oBAAoB;EACzB,MAAM,EAAC,WAAW,mBAAmB,eAAc;EACnD,KAAK,MAAM,aAAa,KAAK,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAK,YAAY,OAAO,SAAS;EAGrC,KAAK,QAAQ,KAAK;GAChB,WAAW;GACX,cAAc,WAAW,QAAQ;EACnC,CAAC;EACD,KAAK,iBAAiB;CACxB;CAEA,MAAM,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAK,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,CAAC,CAAC,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAK,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAK,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAK,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAK,QAAQ,KAAK,GAAG;CACvB;;;;;;;;;;CAWA,MAAM,8BAA8B;EAClC,IAAI,KAAK,4BAA4B,QAAQ,KAAK,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAK;EAClC,IAAI,UAAU,KAAK,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAK,yBAAyB,GACtC;EAEF,KAAK,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAK,YAAY,CAAC,CAAC;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAK,WAAW,UACnD,UAAU,+BACV,KAAK,sBAAsB,EAAE,kNAGS,KAAK,WAAW,EAC7D;EACA,KAAK,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAK,QAAQ,KAAK,GAAG;CACvB;;;;;CAMA,iBACE,QAC8C;EAC9C,IAAI,MAAM;EACV,IAAI;EACJ,KAAK,MAAM,CAAC,WAAW,eAAe,KAAK,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAAK;GACrD,MAAM;GACN,gBAAgB;EAClB;EAEF,IAAI,CAAC,eACH,OAAO;EAET,OAAO;GAAC,WAAW;GAAK,YAAY;EAAa;CACnD;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAK;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;AACF"}

package/out/zero-cache/src/services/change-streamer/replica-poller.js.map

@@ -1 +1 @@
-{"version":3,"file":"replica-poller.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/replica-poller.ts"],"sourcesContent":["import {assert} from 'console';\nimport {clearInterval} from 'timers';\nimport type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nconst CHECK_INTERVAL_MS = 10 * 1000;\n\n/**\n * The single-node equivalent of a backup monitor polls the replica file every\n * 30 seconds and schedules cleanup when the watermark\n * (i.e. stateVersion) moves forward.\n */\nexport class ReplicaPoller {\n  readonly id = 'replica-poller';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #stream: Subscription<BackedUpWatermark>;\n  #pollReplicaTimer: NodeJS.Timeout | undefined;\n\n  #lastWatermark: string = '';\n\n  constructor(lc: LogContext, replicaFile: string) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => clearInterval(this.#pollReplicaTimer),\n    });\n  }\n\n  start(): Source<BackedUpWatermark> {\n    assert(this.#pollReplicaTimer === undefined, `Already called start()`);\n    this.#lc.info?.(`starting replica monitor`);\n    // Perform the first poll immediately to avoid unnecessarily waiting\n    // during single-node development.\n    this.#pollReplica();\n    this.#pollReplicaTimer = setInterval(this.#pollReplica, CHECK_INTERVAL_MS);\n    return this.#stream;\n  }\n\n  readonly #pollReplica = () => {\n    const db = new Database(this.#lc, this.#replicaFile);\n    try {\n      const {stateVersion} = getReplicationState(new StatementRunner(db));\n      if (stateVersion !== this.#lastWatermark) {\n        this.#lastWatermark = stateVersion;\n        this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);\n        this.#stream.push({watermark: stateVersion, backupTimeMs: Date.now()});\n      }\n    } catch (e) {\n      this.#lc.error?.(`Unable to read watermark from replica`, e);\n    } finally {\n      db.close();\n    }\n  };\n}\n"],"mappings":";;;;;;;AAUA,IAAM,oBAAoB;;;;;;AAO1B,IAAa,gBAAb,MAA2B;CACzB,KAAc;CACd;CACA;CACA;CACA;CAEA,iBAAyB;CAEzB,YAAY,IAAgB,aAAqB;EAC/C,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAK,eAAe;EACpB,KAAK,UAAU,aAAa,OAA0B,EACpD,eAAe,cAAc,KAAK,iBAAiB,EACrD,CAAC;CACH;CAEA,QAAmC;EACjC,OAAO,KAAK,sBAAsB,KAAA,GAAW,wBAAwB;EACrE,KAAK,IAAI,OAAO,0BAA0B;EAG1C,KAAK,aAAa;EAClB,KAAK,oBAAoB,YAAY,KAAK,cAAc,iBAAiB;EACzE,OAAO,KAAK;CACd;CAEA,qBAA8B;EAC5B,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,YAAY;EACnD,IAAI;GACF,MAAM,EAAC,iBAAgB,oBAAoB,IAAI,gBAAgB,EAAE,CAAC;GAClE,IAAI,iBAAiB,KAAK,gBAAgB;IACxC,KAAK,iBAAiB;IACtB,KAAK,IAAI,QAAQ,8BAA8B,cAAc;IAC7D,KAAK,QAAQ,KAAK;KAAC,WAAW;KAAc,cAAc,KAAK,IAAI;IAAC,CAAC;GACvE;EACF,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,yCAAyC,CAAC;EAC7D,UAAU;GACR,GAAG,MAAM;EACX;CACF;AACF"}
+{"version":3,"file":"replica-poller.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/replica-poller.ts"],"sourcesContent":["import {assert} from 'console';\nimport {clearInterval} from 'timers';\nimport type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport type {BackedUpWatermark, BackupPoller} from './backup-monitor.ts';\n\nconst CHECK_INTERVAL_MS = 10 * 1000;\n\n/**\n * The single-node equivalent of a backup monitor polls the replica file every\n * 30 seconds and schedules cleanup when the watermark\n * (i.e. stateVersion) moves forward.\n */\nexport class ReplicaPoller implements BackupPoller {\n  readonly id = 'replica-poller';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #stream: Subscription<BackedUpWatermark>;\n  #pollReplicaTimer: NodeJS.Timeout | undefined;\n\n  #lastWatermark: string = '';\n\n  constructor(lc: LogContext, replicaFile: string) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => clearInterval(this.#pollReplicaTimer),\n    });\n  }\n\n  start(): Source<BackedUpWatermark> {\n    assert(this.#pollReplicaTimer === undefined, `Already called start()`);\n    this.#lc.info?.(`starting replica monitor`);\n    // Perform the first poll immediately to avoid unnecessarily waiting\n    // during single-node development.\n    this.#pollReplica();\n    this.#pollReplicaTimer = setInterval(this.#pollReplica, CHECK_INTERVAL_MS);\n    return this.#stream;\n  }\n\n  readonly #pollReplica = () => {\n    const db = new Database(this.#lc, this.#replicaFile);\n    try {\n      const {stateVersion} = getReplicationState(new StatementRunner(db));\n      if (stateVersion !== this.#lastWatermark) {\n        this.#lastWatermark = stateVersion;\n        this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);\n        this.#stream.push({watermark: stateVersion, backupTimeMs: Date.now()});\n      }\n    } catch (e) {\n      this.#lc.error?.(`Unable to read watermark from replica`, e);\n    } finally {\n      db.close();\n    }\n  };\n}\n"],"mappings":";;;;;;;AAUA,IAAM,oBAAoB;;;;;;AAO1B,IAAa,gBAAb,MAAmD;CACjD,KAAc;CACd;CACA;CACA;CACA;CAEA,iBAAyB;CAEzB,YAAY,IAAgB,aAAqB;EAC/C,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAK,eAAe;EACpB,KAAK,UAAU,aAAa,OAA0B,EACpD,eAAe,cAAc,KAAK,iBAAiB,EACrD,CAAC;CACH;CAEA,QAAmC;EACjC,OAAO,KAAK,sBAAsB,KAAA,GAAW,wBAAwB;EACrE,KAAK,IAAI,OAAO,0BAA0B;EAG1C,KAAK,aAAa;EAClB,KAAK,oBAAoB,YAAY,KAAK,cAAc,iBAAiB;EACzE,OAAO,KAAK;CACd;CAEA,qBAA8B;EAC5B,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,YAAY;EACnD,IAAI;GACF,MAAM,EAAC,iBAAgB,oBAAoB,IAAI,gBAAgB,EAAE,CAAC;GAClE,IAAI,iBAAiB,KAAK,gBAAgB;IACxC,KAAK,iBAAiB;IACtB,KAAK,IAAI,QAAQ,8BAA8B,cAAc;IAC7D,KAAK,QAAQ,KAAK;KAAC,WAAW;KAAc,cAAc,KAAK,IAAI;IAAC,CAAC;GACvE;EACF,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,yCAAyC,CAAC;EAC7D,UAAU;GACR,GAAG,MAAM;EACX;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/server-start-metrics.js

@@ -0,0 +1,42 @@
+import { getOrCreateGauge } from "../../observability/metrics.js";
+//#region ../zero-cache/src/services/change-streamer/server-start-metrics.ts
+/**
+* Records where a replication-manager's start goes, reported once it is
+* ready by the `zero.replication.server_start` gauge, in seconds, labeled
+* by `phase` and `kind` (see {@link InitializeKind}). This is for diagnosis,
+* e.g. of why a start is slow, as the process only starts once.
+*/
+var ServerStartMetrics = class {
+	#phasesMs = /* @__PURE__ */ new Map();
+	#kind;
+	#ready = false;
+	setKind(kind) {
+		this.#kind = kind;
+	}
+	record(phase, ms) {
+		if (ms !== void 0 && !this.#ready) this.#phasesMs.set(phase, ms);
+	}
+	/**
+	* Records the `total` (by default, the time since the process started) and
+	* starts reporting the gauge. Phases recorded afterwards are ignored.
+	*/
+	ready(totalMs = performance.now()) {
+		if (this.#ready) return;
+		this.record("total", totalMs);
+		this.#ready = true;
+		const kind = this.#kind ?? "unknown";
+		getOrCreateGauge("replication", "server_start", {
+			description: "The duration of each phase of the replication-manager start (slot_create, fork_wait, restore, catch_up, backup_confirm, and the total from process start to readiness), by the kind of initialization (fork, resume, rmv1, initial_sync).",
+			unit: "s"
+		}).addCallback((o) => {
+			for (const [phase, ms] of this.#phasesMs) o.observe(ms / 1e3, {
+				phase,
+				kind
+			});
+		});
+	}
+};
+//#endregion
+export { ServerStartMetrics };
+
+//# sourceMappingURL=server-start-metrics.js.map

package/out/zero-cache/src/services/change-streamer/server-start-metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"server-start-metrics.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/server-start-metrics.ts"],"sourcesContent":["import {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {InitializeKind} from '../change-source/common/replica-restore.ts';\n\n/**\n * The phases of a replication-manager's start:\n * - `slot_create`: creating the new replica's slot, including retries while\n *   it is blocked by upstream transactions. This is separate from `fork_wait`\n *   because the upstream (e.g. a long-running transaction), not zero-cache or\n *   its capacity, determines how long it takes.\n * - `fork_wait`: selecting the replica to restore from (and claiming its\n *   slot), including the wait for a forked replica to pass the new slot's\n *   position, excluding `slot_create`\n * - `restore`: restoring the replica from its backup\n * - `catch_up`: from initialization to the backup-replicator being caught up\n *   (i.e. its replication lag readiness)\n * - `backup_confirm`: waiting for the first backup to cover the replica\n * - `total`: from the start of the (change-streamer) process to readiness\n *\n * The phases are not exhaustive (e.g. connecting to and preparing the\n * upstream), and `catch_up` and `backup_confirm` overlap.\n */\nexport type ServerStartPhase =\n  | 'slot_create'\n  | 'fork_wait'\n  | 'restore'\n  | 'catch_up'\n  | 'backup_confirm'\n  | 'total';\n\n/**\n * Records where a replication-manager's start goes, reported once it is\n * ready by the `zero.replication.server_start` gauge, in seconds, labeled\n * by `phase` and `kind` (see {@link InitializeKind}). This is for diagnosis,\n * e.g. of why a start is slow, as the process only starts once.\n */\nexport class ServerStartMetrics {\n  readonly #phasesMs = new Map<ServerStartPhase, number>();\n  #kind: InitializeKind | undefined;\n  #ready = false;\n\n  setKind(kind: InitializeKind | undefined) {\n    this.#kind = kind;\n  }\n\n  record(phase: ServerStartPhase, ms: number | undefined) {\n    if (ms !== undefined && !this.#ready) {\n      this.#phasesMs.set(phase, ms);\n    }\n  }\n\n  /**\n   * Records the `total` (by default, the time since the process started) and\n   * starts reporting the gauge. Phases recorded afterwards are ignored.\n   */\n  ready(totalMs = performance.now()) {\n    if (this.#ready) {\n      return;\n    }\n    this.record('total', totalMs);\n    this.#ready = true;\n    const kind = this.#kind ?? 'unknown';\n    getOrCreateGauge('replication', 'server_start', {\n      description:\n        'The duration of each phase of the replication-manager start ' +\n        '(slot_create, fork_wait, restore, catch_up, backup_confirm, and the total ' +\n        'from process start to readiness), by the kind of initialization ' +\n        '(fork, resume, rmv1, initial_sync).',\n      unit: 's',\n    }).addCallback(o => {\n      for (const [phase, ms] of this.#phasesMs) {\n        o.observe(ms / 1000, {phase, kind});\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;;AAmCA,IAAa,qBAAb,MAAgC;CAC9B,4BAAqB,IAAI,IAA8B;CACvD;CACA,SAAS;CAET,QAAQ,MAAkC;EACxC,KAAK,QAAQ;CACf;CAEA,OAAO,OAAyB,IAAwB;EACtD,IAAI,OAAO,KAAA,KAAa,CAAC,KAAK,QAC5B,KAAK,UAAU,IAAI,OAAO,EAAE;CAEhC;;;;;CAMA,MAAM,UAAU,YAAY,IAAI,GAAG;EACjC,IAAI,KAAK,QACP;EAEF,KAAK,OAAO,SAAS,OAAO;EAC5B,KAAK,SAAS;EACd,MAAM,OAAO,KAAK,SAAS;EAC3B,iBAAiB,eAAe,gBAAgB;GAC9C,aACE;GAIF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK;GAClB,KAAK,MAAM,CAAC,OAAO,OAAO,KAAK,WAC7B,EAAE,QAAQ,KAAK,KAAM;IAAC;IAAO;GAAI,CAAC;EAEtC,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/subscriber.js

@@ -40,7 +40,7 @@ var Subscriber = class {
 	#backfills;
 	#onAligned;
 	#onBackfillIgnored;
-	#lagReportingDisabled;
+	#lagReports;
 	#aligned = false;
 	#caughtUpWatermark;
 	#caughtUp = resolver();
@@ -59,7 +59,7 @@ var Subscriber = class {
 		this.#backfills = options.backfills;
 		this.#onAligned = options.onAligned;
 		this.#onBackfillIgnored = options.onBackfillIgnored;
-		this.#lagReportingDisabled = options.lagReportingDisabled;
+		this.#lagReports = options.lagReports;
 	}
 	/**
 	* The subscriber's tracked pending backfills, or `undefined` if the
@@ -426,8 +426,9 @@ var Subscriber = class {
 						this.#onAligned?.(this);
 						this.#caughtUpWatermark = this.#watermark;
 						this.#checkCaughtUp();
-						this.#lagReportingDisabled?.then((disabled) => {
-							if (disabled) this.sendStatus(LAG_REPORTING_DISABLED);
+						this.#lagReports?.then((lagReports) => {
+							if (lagReports) lagReports.request();
+							else this.sendStatus(LAG_REPORTING_DISABLED);
 						});
 					}
 					break;

package/out/zero-cache/src/services/change-streamer/subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n  wsBatched?: boolean | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n\n  /**\n   * The subscriber's pending backfills (protocol v8+), which are tracked\n   * over the changes sent to the subscriber.\n   */\n  backfills?: BackfillState | undefined;\n\n  /**\n   * Called once the subscriber has been sent all changes up to the head of\n   * the stream, i.e. its catchup and backlog have been flushed. From this\n   * point on, the subscriber's tracked {@link backfills} reflect the same\n   * position in the stream as that of the change-streamer.\n   */\n  onAligned?: ((subscriber: Subscriber) => void) | undefined;\n\n  /**\n   * Called when a `backfill` or `backfill-completed` message is ignored for\n   * some of the subscriber's pending backfill columns, after the subscriber\n   * is aligned.\n   */\n  onBackfillIgnored?: ((columns: string[]) => void) | undefined;\n\n  /**\n   * Resolves to whether lag reporting is disabled upstream. If so, a\n   * {@link LAG_REPORTING_DISABLED} status is sent once the subscriber is\n   * aligned, as a stand-in for the lag reports that a subscriber would\n   * otherwise use to determine that it is caught up (e.g. to advertise\n   * readiness to serve traffic).\n   */\n  lagReportingDisabled?: Promise<boolean> | undefined;\n\n  /**\n   * Whether the subscriber switches over to another replication-manager\n   * when this one drains (see `DrainTracker`).\n   */\n  drainable?: boolean | undefined;\n};\n\n/**\n * A status message queued in the backlog, so that it is sent downstream in\n * the order in which it was received relative to (backlogged) changes.\n */\ntype QueuedStatus = {readonly status: string};\n\ntype BacklogEntry = WatermarkedChange | QueuedStatus;\n\nfunction isQueuedStatus(entry: BacklogEntry): entry is QueuedStatus {\n  return !Array.isArray(entry);\n}\n\nfunction entryBytes(entry: BacklogEntry): number {\n  return isQueuedStatus(entry) ? entry.status.length : entry[2].length;\n}\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly drainable: boolean;\n  readonly #downstream: Subscription<string | PreSerializedBatch>;\n  readonly #wsBatched: boolean;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<BacklogEntry> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n  readonly #backfills: BackfillState | undefined;\n  readonly #onAligned: ((subscriber: Subscriber) => void) | undefined;\n  readonly #onBackfillIgnored: ((columns: string[]) => void) | undefined;\n  readonly #lagReportingDisabled: Promise<boolean> | undefined;\n  #aligned = false;\n  // The watermark to be acked for the subscriber to be caught up, set when\n  // it is aligned.\n  #caughtUpWatermark: string | undefined;\n  readonly #caughtUp = resolver<void>();\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string | PreSerializedBatch>,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.drainable = options.drainable ?? false;\n    this.#downstream = downstream;\n    this.#wsBatched = options.wsBatched ?? false;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n    this.#backfills = options.backfills;\n    this.#onAligned = options.onAligned;\n    this.#onBackfillIgnored = options.onBackfillIgnored;\n    this.#lagReportingDisabled = options.lagReportingDisabled;\n  }\n\n  /**\n   * The subscriber's tracked pending backfills, or `undefined` if the\n   * subscriber does not report them (i.e. protocol < v8).\n   */\n  get backfills(): BackfillState | undefined {\n    return this.#backfills;\n  }\n\n  /**\n   * Whether the subscriber has been sent all changes up to the head of the\n   * stream (see {@link SubscriberOptions.onAligned}).\n   */\n  get aligned(): boolean {\n    return this.#aligned;\n  }\n\n  /**\n   * Resolves once the subscriber has caught up, i.e. once it has acked\n   * (durably applied) all changes up to the head of the stream at the time\n   * it was {@link aligned}. Never resolves if the subscriber is closed first.\n   */\n  get caughtUp(): Promise<void> {\n    return this.#caughtUp.promise;\n  }\n\n  #trackBackfills(change: WatermarkedChange) {\n    if (this.#backfills) {\n      const ignored = this.#backfills.applySerialized(change);\n      if (ignored.length && this.#aligned) {\n        this.#onBackfillIgnored?.(ignored);\n      }\n    }\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  sendBatch(\n    changes: readonly WatermarkedChange[],\n    preSerialized?: PreSerializedBatch | undefined,\n  ): Promise<void> {\n    if (changes.length === 0) {\n      return promiseVoid;\n    }\n\n    if (\n      this.#wsBatched &&\n      preSerialized &&\n      !this.#backlog &&\n      this.#initialized &&\n      changes[0][0] > this.#watermark &&\n      (this.#protocolVersion >= 5 ||\n        changes.every(c => this.supportsMessage(c[1])))\n    ) {\n      let commitWatermark: string | undefined;\n      for (let i = changes.length - 1; i >= 0; i--) {\n        if (changes[i][1] === 'commit') {\n          commitWatermark = changes[i][0];\n          break;\n        }\n      }\n      if (commitWatermark) {\n        this.#watermark = commitWatermark;\n      }\n      for (const change of changes) {\n        this.#trackBackfills(change);\n      }\n      return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n    }\n\n    const promises: Promise<void>[] = [];\n    for (const change of changes) {\n      const p = this.send(change);\n      if (p !== promiseVoid) {\n        promises.push(p);\n      }\n    }\n    if (promises.length === 0) {\n      return promiseVoid;\n    }\n    if (promises.length === 1) {\n      return promises[0];\n    }\n    return Promise.all(promises).then(() => {});\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      // The initial status precedes the catchup, so it bypasses the backlog.\n      void this.#sendDownstream(['status', {tag: 'status'}]);\n    }\n  }\n\n  /**\n   * Sends a status message (e.g. a lag report) downstream. While the\n   * subscriber is catching up, the status is queued in the backlog behind\n   * the changes that preceded it in the replication stream, so that the\n   * subscriber processes it in stream order. This is necessary for lag\n   * reports to measure the lag of the subscriber, including its catchup.\n   *\n   * Status messages received before the subscriber is initialized are\n   * dropped.\n   */\n  sendStatus(status: Status) {\n    if (!this.#initialized) {\n      return;\n    }\n    const downstream: Downstream = ['status', status];\n    if (this.#backlog) {\n      this.#pushBacklog({status: BigIntJSON.stringify(downstream)});\n      return;\n    }\n    void this.#sendDownstream(downstream);\n  }\n\n  /**\n   * Sends a control status (e.g. `draining` or `restore`) downstream\n   * immediately, regardless of whether the subscriber is initialized or\n   * catching up. Unlike {@link sendStatus}, such statuses are not tied to a\n   * position in the replication stream.\n   */\n  sendControlStatus(status: Status) {\n    void this.#sendDownstream(['status', status]);\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    this.#trackBackfills(change);\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      this.#advanceAck(watermark);\n    }\n  }\n\n  #advanceAck(watermark: string) {\n    // Sends can complete out of order (e.g. the bounded window in\n    // #drainBacklog), so the ack only advances monotonically, and listeners\n    // are only notified when it does.\n    const acked = max(this.#acked, watermark);\n    if (acked !== this.#acked) {\n      this.#acked = acked;\n      this.#onAck?.(acked);\n      this.#checkCaughtUp();\n    }\n  }\n\n  #checkCaughtUp() {\n    if (\n      this.#caughtUpWatermark !== undefined &&\n      this.#acked >= this.#caughtUpWatermark\n    ) {\n      this.#caughtUpWatermark = undefined;\n      this.#caughtUp.resolve();\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  async #sendPreSerializedDownstream(\n    batch: PreSerializedBatch,\n    commitWatermark?: string | undefined,\n  ): Promise<void> {\n    const size = batch.byteLength;\n    this.#pending += batch.changes.length;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(batch);\n    try {\n      const outcome = await result;\n      if (commitWatermark && outcome === 'consumed') {\n        this.#advanceAck(commitWatermark);\n      }\n    } finally {\n      this.#pending -= batch.changes.length;\n      this.#pendingBytes -= size;\n      this.#processed += batch.changes.length;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(\n    error?: ErrorType,\n    message?: string,\n    details?: {replicaVersion?: string | undefined},\n  ) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream([\n        'error',\n        {type: error, message, ...details},\n      ]).finally(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(entry: BacklogEntry) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(entry);\n    this.#backlogBytes += entryBytes(entry);\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const entry = this.#backlog?.shift();\n        if (!entry) {\n          const closed = this.#backlog === null;\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          if (!closed && !this.#aligned) {\n            // All changes up to the head have been handed to #sendChange()\n            // (and thus tracked), and subsequent changes are sent directly.\n            this.#aligned = true;\n            this.#onAligned?.(this);\n            this.#caughtUpWatermark = this.#watermark;\n            this.#checkCaughtUp();\n            void this.#lagReportingDisabled?.then(disabled => {\n              if (disabled) {\n                this.sendStatus(LAG_REPORTING_DISABLED);\n              }\n            });\n          }\n          break;\n        }\n\n        const bytes = entryBytes(entry);\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const send = isQueuedStatus(entry)\n          ? this.#sendStringifiedDownstream(entry.status).then(() => {})\n          : this.#sendChange(entry);\n        const promise = send.finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n\n/**\n * Sent to an aligned subscriber when lag reporting is disabled upstream,\n * signaling that the subscriber is caught up. This is a lag report with a\n * `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured\n * (see `isLagReportingDisabledSignal()` in the replicator's recorder).\n */\nexport const LAG_REPORTING_DISABLED: Status = {\n  tag: 'status',\n  lagReport: {nextSendTimeMs: 0},\n};\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC;AACzC,IAAM,kCAAkC;AA2DxC,SAAS,eAAe,OAA4C;CAClE,OAAO,CAAC,MAAM,QAAQ,KAAK;AAC7B;AAEA,SAAS,WAAW,OAA6B;CAC/C,OAAO,eAAe,KAAK,IAAI,MAAM,OAAO,SAAS,MAAM,EAAE,CAAC;AAChE;;;;;;;;AAuBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA;CACA,WAAW;CAGX;CACA,YAAqB,SAAe;CAEpC,YACE,iBACA,IACA,MACA,WACA,YACA,UAA6B,CAAC,GAC9B;EACA,KAAK,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAK,YAAY,QAAQ,aAAa;EACtC,KAAK,cAAc;EACnB,KAAK,aAAa,QAAQ,aAAa;EACvC,KAAK,aAAa;EAClB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,WAAW;EAC/B,KAAK,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAK,SAAS,QAAQ;EACtB,KAAK,aAAa,QAAQ;EAC1B,KAAK,aAAa,QAAQ;EAC1B,KAAK,qBAAqB,QAAQ;EAClC,KAAK,wBAAwB,QAAQ;CACvC;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAK;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAK;CACd;;;;;;CAOA,IAAI,WAA0B;EAC5B,OAAO,KAAK,UAAU;CACxB;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAK,YAAY;GACnB,MAAM,UAAU,KAAK,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAK,UACzB,KAAK,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAK;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAK;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAK,yBAAyB,KAAK,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAK,aAAa,QAAQ,KAAK,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAK,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAK,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAK,YAAY;GAC/B,IAAI,KAAK,UAAU;IAIjB,KAAK,aAAa,MAAM;IACxB,OAAO,KAAK,0BAA0B;GACxC;GACA,OAAO,KAAK,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAK,cACL,iBACA,CAAC,KAAK,YACN,KAAK,gBACL,QAAQ,EAAE,CAAC,KAAK,KAAK,eACpB,KAAK,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,EAAE,CAAC,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,EAAE,CAAC;IAC7B;GACF;GAEF,IAAI,iBACF,KAAK,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAK,gBAAgB,MAAM;GAE7B,OAAO,KAAK,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAK,cAAc;GACtB,KAAK,eAAe;GAEpB,KAAU,gBAAgB,CAAC,UAAU,EAAC,KAAK,SAAQ,CAAC,CAAC;EACvD;CACF;;;;;;;;;;;CAYA,WAAW,QAAgB;EACzB,IAAI,CAAC,KAAK,cACR;EAEF,MAAM,aAAyB,CAAC,UAAU,MAAM;EAChD,IAAI,KAAK,UAAU;GACjB,KAAK,aAAa,EAAC,QAAQ,WAAW,UAAU,UAAU,EAAC,CAAC;GAC5D;EACF;EACA,KAAU,gBAAgB,UAAU;CACtC;;;;;;;CAQA,kBAAkB,QAAgB;EAChC,KAAU,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAC9C;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAK,YAAY;EACjB,MAAM,KAAK,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAK,YAAY;EACjB,IAAI,CAAC,KAAK,UACR,OAAO,KAAK,iBAAiB;EAE/B,IAAI,CAAC,KAAK,eAAe;GAKvB,KAAK,gBAAgB,KAAK,cAAc;GACxC,KAAU,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAK;CACd;CAEA,MAAM,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAK,aAAa;EAEpB,KAAK,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAK,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YACjC,KAAK,YAAY,SAAS;CAE9B;CAEA,YAAY,WAAmB;EAI7B,MAAM,QAAQ,IAAI,KAAK,QAAQ,SAAS;EACxC,IAAI,UAAU,KAAK,QAAQ;GACzB,KAAK,SAAS;GACd,KAAK,SAAS,KAAK;GACnB,KAAK,eAAe;EACtB;CACF;CAEA,iBAAiB;EACf,IACE,KAAK,uBAAuB,KAAA,KAC5B,KAAK,UAAU,KAAK,oBACpB;GACA,KAAK,qBAAqB,KAAA;GAC1B,KAAK,UAAU,QAAQ;EACzB;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAK,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAM,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAK;EACL,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAK;GACL,KAAK,iBAAiB;GACtB,KAAK;EACP;CACF;CAEA,MAAM,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAK,YAAY,MAAM,QAAQ;EAC/B,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YACjC,KAAK,YAAY,eAAe;EAEpC,UAAU;GACR,KAAK,YAAY,MAAM,QAAQ;GAC/B,KAAK,iBAAiB;GACtB,KAAK,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAK,WAAW,KAAK;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAK;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAK,SAAS,UAAU,YAC7B,KAAK,SAAS,MAAM;EAEtB,KAAK,SAAS,KAAK;GAAC,WAAW,KAAK;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAK,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;EAEF,MAAM,OAAO,KAAK,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAK,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAK,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAK,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAK,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAK,oBAAoB;EACzB,OAAO,MAAM,KAAK;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAK,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MACE,OACA,SACA,SACA;EAGA,KAAK,WAAW;EAChB,KAAK,gBAAgB;EACrB,KAAK,qBAAqB,WAAW;EAGrC,KAAK,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAU,gBAAgB,CACxB,SACA;GAAC,MAAM;GAAO;GAAS,GAAG;EAAO,CACnC,CAAC,CAAC,CAAC,cAAc,KAAK,YAAY,OAAO,CAAC;OAE1C,KAAK,YAAY,OAAO;CAE5B;CAEA,IAAI,gBAAgB;EAClB,OAAO,KAAK,UAAU,QAAQ;CAChC;CAEA,IAAI,wBAAwB;EAI1B,OAAO,KAAK,gBAAgB,KAAK;CACnC;CAEA,IAAI,sBAAsB;EACxB,OAAO,KAAK,wBAAwB,KAAK;CAC3C;CAEA,aAAa,OAAqB;EAChC,OAAO,KAAK,UAAU,gDAAgD;EACtE,KAAK,SAAS,KAAK,KAAK;EACxB,KAAK,iBAAiB,WAAW,KAAK;EACtC,IAAI,KAAK,aACP,KAAK,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAK,mBAAmB;EAC5C,KAAK,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAK,qBAAqB,aAAa,KAAK,qBAAqB;CAC1E;CAEA,MAAM,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,QAAQ,KAAK,UAAU,MAAM;IACnC,IAAI,CAAC,OAAO;KACV,MAAM,SAAS,KAAK,aAAa;KACjC,KAAK,WAAW;KAChB,KAAK,gBAAgB;KACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAK,UAAU;MAG7B,KAAK,WAAW;MAChB,KAAK,aAAa,IAAI;MACtB,KAAK,qBAAqB,KAAK;MAC/B,KAAK,eAAe;MACpB,KAAU,uBAAuB,MAAK,aAAY;OAChD,IAAI,UACF,KAAK,WAAW,sBAAsB;MAE1C,CAAC;KACH;KACA;IACF;IAEA,MAAM,QAAQ,WAAW,KAAK;IAC9B,KAAK,iBAAiB;IACtB,KAAK,yBAAyB;IAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IAQA,MAAM,WAHO,eAAe,KAAK,IAC7B,KAAK,2BAA2B,MAAM,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC,IAC3D,KAAK,YAAY,KAAK,EAAA,CACL,cAAc;KACjC,KAAK,yBAAyB;KAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAK,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAK,gBAAgB;GACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAK,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAK,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAK,SAAS,WAAW,KAAK,gBAAgB,KAAK,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAK,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF;;;;;;;AAQA,IAAa,yBAAiC;CAC5C,KAAK;CACL,WAAW,EAAC,gBAAgB,EAAC;AAC/B"}
+{"version":3,"file":"subscriber.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n  wsBatched?: boolean | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n\n  /**\n   * The subscriber's pending backfills (protocol v8+), which are tracked\n   * over the changes sent to the subscriber.\n   */\n  backfills?: BackfillState | undefined;\n\n  /**\n   * Called once the subscriber has been sent all changes up to the head of\n   * the stream, i.e. its catchup and backlog have been flushed. From this\n   * point on, the subscriber's tracked {@link backfills} reflect the same\n   * position in the stream as that of the change-streamer.\n   */\n  onAligned?: ((subscriber: Subscriber) => void) | undefined;\n\n  /**\n   * Called when a `backfill` or `backfill-completed` message is ignored for\n   * some of the subscriber's pending backfill columns, after the subscriber\n   * is aligned.\n   */\n  onBackfillIgnored?: ((columns: string[]) => void) | undefined;\n\n  /**\n   * Resolves, once it is known whether lag reporting is enabled upstream, to\n   * the means of requesting a lag report, or to `undefined` if it is disabled.\n   *\n   * Subscribers use lag reports to determine that they are caught up (e.g. to\n   * advertise readiness to serve traffic). Once the subscriber is aligned, a\n   * report is therefore requested, so that it is received without waiting out\n   * the report interval. If lag reporting is disabled, a\n   * {@link LAG_REPORTING_DISABLED} status is sent as a stand-in instead.\n   */\n  lagReports?: Promise<LagReports | undefined> | undefined;\n\n  /**\n   * Whether the subscriber switches over to another replication-manager\n   * when this one drains (see `DrainTracker`).\n   */\n  drainable?: boolean | undefined;\n};\n\n/**\n * A status message queued in the backlog, so that it is sent downstream in\n * the order in which it was received relative to (backlogged) changes.\n */\ntype QueuedStatus = {readonly status: string};\n\ntype BacklogEntry = WatermarkedChange | QueuedStatus;\n\nfunction isQueuedStatus(entry: BacklogEntry): entry is QueuedStatus {\n  return !Array.isArray(entry);\n}\n\nfunction entryBytes(entry: BacklogEntry): number {\n  return isQueuedStatus(entry) ? entry.status.length : entry[2].length;\n}\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly drainable: boolean;\n  readonly #downstream: Subscription<string | PreSerializedBatch>;\n  readonly #wsBatched: boolean;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<BacklogEntry> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n  readonly #backfills: BackfillState | undefined;\n  readonly #onAligned: ((subscriber: Subscriber) => void) | undefined;\n  readonly #onBackfillIgnored: ((columns: string[]) => void) | undefined;\n  readonly #lagReports: Promise<LagReports | undefined> | undefined;\n  #aligned = false;\n  // The watermark to be acked for the subscriber to be caught up, set when\n  // it is aligned.\n  #caughtUpWatermark: string | undefined;\n  readonly #caughtUp = resolver<void>();\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string | PreSerializedBatch>,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.drainable = options.drainable ?? false;\n    this.#downstream = downstream;\n    this.#wsBatched = options.wsBatched ?? false;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n    this.#backfills = options.backfills;\n    this.#onAligned = options.onAligned;\n    this.#onBackfillIgnored = options.onBackfillIgnored;\n    this.#lagReports = options.lagReports;\n  }\n\n  /**\n   * The subscriber's tracked pending backfills, or `undefined` if the\n   * subscriber does not report them (i.e. protocol < v8).\n   */\n  get backfills(): BackfillState | undefined {\n    return this.#backfills;\n  }\n\n  /**\n   * Whether the subscriber has been sent all changes up to the head of the\n   * stream (see {@link SubscriberOptions.onAligned}).\n   */\n  get aligned(): boolean {\n    return this.#aligned;\n  }\n\n  /**\n   * Resolves once the subscriber has caught up, i.e. once it has acked\n   * (durably applied) all changes up to the head of the stream at the time\n   * it was {@link aligned}. Never resolves if the subscriber is closed first.\n   */\n  get caughtUp(): Promise<void> {\n    return this.#caughtUp.promise;\n  }\n\n  #trackBackfills(change: WatermarkedChange) {\n    if (this.#backfills) {\n      const ignored = this.#backfills.applySerialized(change);\n      if (ignored.length && this.#aligned) {\n        this.#onBackfillIgnored?.(ignored);\n      }\n    }\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  sendBatch(\n    changes: readonly WatermarkedChange[],\n    preSerialized?: PreSerializedBatch | undefined,\n  ): Promise<void> {\n    if (changes.length === 0) {\n      return promiseVoid;\n    }\n\n    if (\n      this.#wsBatched &&\n      preSerialized &&\n      !this.#backlog &&\n      this.#initialized &&\n      changes[0][0] > this.#watermark &&\n      (this.#protocolVersion >= 5 ||\n        changes.every(c => this.supportsMessage(c[1])))\n    ) {\n      let commitWatermark: string | undefined;\n      for (let i = changes.length - 1; i >= 0; i--) {\n        if (changes[i][1] === 'commit') {\n          commitWatermark = changes[i][0];\n          break;\n        }\n      }\n      if (commitWatermark) {\n        this.#watermark = commitWatermark;\n      }\n      for (const change of changes) {\n        this.#trackBackfills(change);\n      }\n      return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n    }\n\n    const promises: Promise<void>[] = [];\n    for (const change of changes) {\n      const p = this.send(change);\n      if (p !== promiseVoid) {\n        promises.push(p);\n      }\n    }\n    if (promises.length === 0) {\n      return promiseVoid;\n    }\n    if (promises.length === 1) {\n      return promises[0];\n    }\n    return Promise.all(promises).then(() => {});\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      // The initial status precedes the catchup, so it bypasses the backlog.\n      void this.#sendDownstream(['status', {tag: 'status'}]);\n    }\n  }\n\n  /**\n   * Sends a status message (e.g. a lag report) downstream. While the\n   * subscriber is catching up, the status is queued in the backlog behind\n   * the changes that preceded it in the replication stream, so that the\n   * subscriber processes it in stream order. This is necessary for lag\n   * reports to measure the lag of the subscriber, including its catchup.\n   *\n   * Status messages received before the subscriber is initialized are\n   * dropped.\n   */\n  sendStatus(status: Status) {\n    if (!this.#initialized) {\n      return;\n    }\n    const downstream: Downstream = ['status', status];\n    if (this.#backlog) {\n      this.#pushBacklog({status: BigIntJSON.stringify(downstream)});\n      return;\n    }\n    void this.#sendDownstream(downstream);\n  }\n\n  /**\n   * Sends a control status (e.g. `draining` or `restore`) downstream\n   * immediately, regardless of whether the subscriber is initialized or\n   * catching up. Unlike {@link sendStatus}, such statuses are not tied to a\n   * position in the replication stream.\n   */\n  sendControlStatus(status: Status) {\n    void this.#sendDownstream(['status', status]);\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    this.#trackBackfills(change);\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      this.#advanceAck(watermark);\n    }\n  }\n\n  #advanceAck(watermark: string) {\n    // Sends can complete out of order (e.g. the bounded window in\n    // #drainBacklog), so the ack only advances monotonically, and listeners\n    // are only notified when it does.\n    const acked = max(this.#acked, watermark);\n    if (acked !== this.#acked) {\n      this.#acked = acked;\n      this.#onAck?.(acked);\n      this.#checkCaughtUp();\n    }\n  }\n\n  #checkCaughtUp() {\n    if (\n      this.#caughtUpWatermark !== undefined &&\n      this.#acked >= this.#caughtUpWatermark\n    ) {\n      this.#caughtUpWatermark = undefined;\n      this.#caughtUp.resolve();\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  async #sendPreSerializedDownstream(\n    batch: PreSerializedBatch,\n    commitWatermark?: string | undefined,\n  ): Promise<void> {\n    const size = batch.byteLength;\n    this.#pending += batch.changes.length;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(batch);\n    try {\n      const outcome = await result;\n      if (commitWatermark && outcome === 'consumed') {\n        this.#advanceAck(commitWatermark);\n      }\n    } finally {\n      this.#pending -= batch.changes.length;\n      this.#pendingBytes -= size;\n      this.#processed += batch.changes.length;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(\n    error?: ErrorType,\n    message?: string,\n    details?: {replicaVersion?: string | undefined},\n  ) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream([\n        'error',\n        {type: error, message, ...details},\n      ]).finally(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(entry: BacklogEntry) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(entry);\n    this.#backlogBytes += entryBytes(entry);\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const entry = this.#backlog?.shift();\n        if (!entry) {\n          const closed = this.#backlog === null;\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          if (!closed && !this.#aligned) {\n            // All changes up to the head have been handed to #sendChange()\n            // (and thus tracked), and subsequent changes are sent directly.\n            this.#aligned = true;\n            this.#onAligned?.(this);\n            this.#caughtUpWatermark = this.#watermark;\n            this.#checkCaughtUp();\n            void this.#lagReports?.then(lagReports => {\n              if (lagReports) {\n                lagReports.request();\n              } else {\n                this.sendStatus(LAG_REPORTING_DISABLED);\n              }\n            });\n          }\n          break;\n        }\n\n        const bytes = entryBytes(entry);\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const send = isQueuedStatus(entry)\n          ? this.#sendStringifiedDownstream(entry.status).then(() => {})\n          : this.#sendChange(entry);\n        const promise = send.finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n\n/** Requests lag reports from upstream (see {@link SubscriberOptions.lagReports}). */\nexport type LagReports = {\n  /**\n   * Requests that a lag report be sent now rather than at the next report\n   * interval. Requests are coalesced upstream, e.g. with a report in flight.\n   */\n  request(): void;\n};\n\n/**\n * Sent to an aligned subscriber when lag reporting is disabled upstream,\n * signaling that the subscriber is caught up. This is a lag report with a\n * `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured\n * (see `isLagReportingDisabledSignal()` in the replicator's recorder).\n */\nexport const LAG_REPORTING_DISABLED: Status = {\n  tag: 'status',\n  lagReport: {nextSendTimeMs: 0},\n};\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC;AACzC,IAAM,kCAAkC;AA8DxC,SAAS,eAAe,OAA4C;CAClE,OAAO,CAAC,MAAM,QAAQ,KAAK;AAC7B;AAEA,SAAS,WAAW,OAA6B;CAC/C,OAAO,eAAe,KAAK,IAAI,MAAM,OAAO,SAAS,MAAM,EAAE,CAAC;AAChE;;;;;;;;AAuBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA;CACA,WAAW;CAGX;CACA,YAAqB,SAAe;CAEpC,YACE,iBACA,IACA,MACA,WACA,YACA,UAA6B,CAAC,GAC9B;EACA,KAAK,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAK,YAAY,QAAQ,aAAa;EACtC,KAAK,cAAc;EACnB,KAAK,aAAa,QAAQ,aAAa;EACvC,KAAK,aAAa;EAClB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,WAAW;EAC/B,KAAK,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAK,SAAS,QAAQ;EACtB,KAAK,aAAa,QAAQ;EAC1B,KAAK,aAAa,QAAQ;EAC1B,KAAK,qBAAqB,QAAQ;EAClC,KAAK,cAAc,QAAQ;CAC7B;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAK;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAK;CACd;;;;;;CAOA,IAAI,WAA0B;EAC5B,OAAO,KAAK,UAAU;CACxB;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAK,YAAY;GACnB,MAAM,UAAU,KAAK,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAK,UACzB,KAAK,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAK;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAK;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAK,yBAAyB,KAAK,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAK,aAAa,QAAQ,KAAK,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAK,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAK,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAK,YAAY;GAC/B,IAAI,KAAK,UAAU;IAIjB,KAAK,aAAa,MAAM;IACxB,OAAO,KAAK,0BAA0B;GACxC;GACA,OAAO,KAAK,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAK,cACL,iBACA,CAAC,KAAK,YACN,KAAK,gBACL,QAAQ,EAAE,CAAC,KAAK,KAAK,eACpB,KAAK,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,EAAE,CAAC,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,EAAE,CAAC;IAC7B;GACF;GAEF,IAAI,iBACF,KAAK,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAK,gBAAgB,MAAM;GAE7B,OAAO,KAAK,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAK,cAAc;GACtB,KAAK,eAAe;GAEpB,KAAU,gBAAgB,CAAC,UAAU,EAAC,KAAK,SAAQ,CAAC,CAAC;EACvD;CACF;;;;;;;;;;;CAYA,WAAW,QAAgB;EACzB,IAAI,CAAC,KAAK,cACR;EAEF,MAAM,aAAyB,CAAC,UAAU,MAAM;EAChD,IAAI,KAAK,UAAU;GACjB,KAAK,aAAa,EAAC,QAAQ,WAAW,UAAU,UAAU,EAAC,CAAC;GAC5D;EACF;EACA,KAAU,gBAAgB,UAAU;CACtC;;;;;;;CAQA,kBAAkB,QAAgB;EAChC,KAAU,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAC9C;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAK,YAAY;EACjB,MAAM,KAAK,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAK,YAAY;EACjB,IAAI,CAAC,KAAK,UACR,OAAO,KAAK,iBAAiB;EAE/B,IAAI,CAAC,KAAK,eAAe;GAKvB,KAAK,gBAAgB,KAAK,cAAc;GACxC,KAAU,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAK;CACd;CAEA,MAAM,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAK,aAAa;EAEpB,KAAK,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAK,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YACjC,KAAK,YAAY,SAAS;CAE9B;CAEA,YAAY,WAAmB;EAI7B,MAAM,QAAQ,IAAI,KAAK,QAAQ,SAAS;EACxC,IAAI,UAAU,KAAK,QAAQ;GACzB,KAAK,SAAS;GACd,KAAK,SAAS,KAAK;GACnB,KAAK,eAAe;EACtB;CACF;CAEA,iBAAiB;EACf,IACE,KAAK,uBAAuB,KAAA,KAC5B,KAAK,UAAU,KAAK,oBACpB;GACA,KAAK,qBAAqB,KAAA;GAC1B,KAAK,UAAU,QAAQ;EACzB;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAK,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAM,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAK;EACL,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAK;GACL,KAAK,iBAAiB;GACtB,KAAK;EACP;CACF;CAEA,MAAM,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAK,YAAY,MAAM,QAAQ;EAC/B,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YACjC,KAAK,YAAY,eAAe;EAEpC,UAAU;GACR,KAAK,YAAY,MAAM,QAAQ;GAC/B,KAAK,iBAAiB;GACtB,KAAK,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAK,WAAW,KAAK;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAK;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAK,SAAS,UAAU,YAC7B,KAAK,SAAS,MAAM;EAEtB,KAAK,SAAS,KAAK;GAAC,WAAW,KAAK;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAK,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;EAEF,MAAM,OAAO,KAAK,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAK,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAK,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAK,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAK,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAK,oBAAoB;EACzB,OAAO,MAAM,KAAK;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAK,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MACE,OACA,SACA,SACA;EAGA,KAAK,WAAW;EAChB,KAAK,gBAAgB;EACrB,KAAK,qBAAqB,WAAW;EAGrC,KAAK,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAU,gBAAgB,CACxB,SACA;GAAC,MAAM;GAAO;GAAS,GAAG;EAAO,CACnC,CAAC,CAAC,CAAC,cAAc,KAAK,YAAY,OAAO,CAAC;OAE1C,KAAK,YAAY,OAAO;CAE5B;CAEA,IAAI,gBAAgB;EAClB,OAAO,KAAK,UAAU,QAAQ;CAChC;CAEA,IAAI,wBAAwB;EAI1B,OAAO,KAAK,gBAAgB,KAAK;CACnC;CAEA,IAAI,sBAAsB;EACxB,OAAO,KAAK,wBAAwB,KAAK;CAC3C;CAEA,aAAa,OAAqB;EAChC,OAAO,KAAK,UAAU,gDAAgD;EACtE,KAAK,SAAS,KAAK,KAAK;EACxB,KAAK,iBAAiB,WAAW,KAAK;EACtC,IAAI,KAAK,aACP,KAAK,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAK,mBAAmB;EAC5C,KAAK,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAK,qBAAqB,aAAa,KAAK,qBAAqB;CAC1E;CAEA,MAAM,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,QAAQ,KAAK,UAAU,MAAM;IACnC,IAAI,CAAC,OAAO;KACV,MAAM,SAAS,KAAK,aAAa;KACjC,KAAK,WAAW;KAChB,KAAK,gBAAgB;KACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAK,UAAU;MAG7B,KAAK,WAAW;MAChB,KAAK,aAAa,IAAI;MACtB,KAAK,qBAAqB,KAAK;MAC/B,KAAK,eAAe;MACpB,KAAU,aAAa,MAAK,eAAc;OACxC,IAAI,YACF,WAAW,QAAQ;YAEnB,KAAK,WAAW,sBAAsB;MAE1C,CAAC;KACH;KACA;IACF;IAEA,MAAM,QAAQ,WAAW,KAAK;IAC9B,KAAK,iBAAiB;IACtB,KAAK,yBAAyB;IAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IAQA,MAAM,WAHO,eAAe,KAAK,IAC7B,KAAK,2BAA2B,MAAM,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC,IAC3D,KAAK,YAAY,KAAK,EAAA,CACL,cAAc;KACjC,KAAK,yBAAyB;KAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAK,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAK,gBAAgB;GACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAK,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAK,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAK,SAAS,WAAW,KAAK,gBAAgB,KAAK,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAK,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF;;;;;;;AAiBA,IAAa,yBAAiC;CAC5C,KAAK;CACL,WAAW,EAAC,gBAAgB,EAAC;AAC/B"}

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -138,8 +138,7 @@ async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, mut
 	}
 	authorizer.reloadPermissions();
 	if (!errorMode) {
-		const { ops } = mutation.args[0];
-		authorizer.validateTableNames(ops);
+		const ops = authorizer.validateOps(mutation.args[0].ops);
 		const normalizedOps = authorizer.normalizeOps(ops);
 		const [canPre, canPost] = await Promise.all([authorizer.canPreMutation(authData, normalizedOps), authorizer.canPostMutation(authData, normalizedOps)]);
 		if (canPre && canPost) for (const op of ops) switch (op.op) {