npm - @rocicorp/zero - Versions diffs - 1.11.0-canary.25 → 1.11.0-canary.26 - Mend

@rocicorp/zero 1.11.0-canary.25 → 1.11.0-canary.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (117) hide show

package/out/zero-cache/src/services/change-streamer/subscriber.js CHANGED Viewed

@@ -8,6 +8,12 @@ import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-streamer/subscriber.ts
 var DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;
 var DEFAULT_BACKLOG_LOW_WATER_RATIO = .8;
+function isQueuedStatus(entry) {
+	return !Array.isArray(entry);
+}
+function entryBytes(entry) {
+	return isQueuedStatus(entry) ? entry.status.length : entry[2].length;
+}
 /**
 * Encapsulates a subscriber to changes. All subscribers start in a
 * "catchup" phase in which changes are buffered in a backlog while the
@@ -20,7 +26,6 @@ var Subscriber = class {
 	id;
 	mode;
 	#downstream;
-	#latestStatus;
 	#wsBatched;
 	#watermark;
 	#acked;
@@ -34,13 +39,13 @@ var Subscriber = class {
 	#backfills;
 	#onAligned;
 	#onBackfillIgnored;
+	#lagReportingDisabled;
 	#aligned = false;
-	constructor(protocolVersion, id, mode, watermark, downstream, latestStatus, options = {}) {
+	constructor(protocolVersion, id, mode, watermark, downstream, options = {}) {
 		this.#protocolVersion = protocolVersion;
 		this.id = id;
 		this.mode = mode;
 		this.#downstream = downstream;
-		this.#latestStatus = latestStatus;
 		this.#wsBatched = options.wsBatched ?? false;
 		this.#watermark = watermark;
 		this.#acked = watermark;
@@ -50,6 +55,7 @@ var Subscriber = class {
 		this.#backfills = options.backfills;
 		this.#onAligned = options.onAligned;
 		this.#onBackfillIgnored = options.onBackfillIgnored;
+		this.#lagReportingDisabled = options.lagReportingDisabled;
 	}
 	/**
 	* The subscriber's tracked pending backfills, or `undefined` if the
@@ -153,11 +159,27 @@ var Subscriber = class {
 	#initialize() {
 		if (!this.#initialized) {
 			this.#initialized = true;
-			this.sendStatus(this.#latestStatus());
+			this.#sendDownstream(["status", { tag: "status" }]);
 		}
 	}
+	/**
+	* Sends a status message (e.g. a lag report) downstream. While the
+	* subscriber is catching up, the status is queued in the backlog behind
+	* the changes that preceded it in the replication stream, so that the
+	* subscriber processes it in stream order. This is necessary for lag
+	* reports to measure the lag of the subscriber, including its catchup.
+	*
+	* Status messages received before the subscriber is initialized are
+	* dropped.
+	*/
 	sendStatus(status) {
-		if (this.#initialized) this.#sendDownstream(["status", status]);
+		if (!this.#initialized) return;
+		const downstream = ["status", status];
+		if (this.#backlog) {
+			this.#pushBacklog({ status: BigIntJSON.stringify(downstream) });
+			return;
+		}
+		this.#sendDownstream(downstream);
 	}
 	/** catchup() is called on ChangeEntries loaded from the store. */
 	async catchup(change) {
@@ -350,10 +372,10 @@ var Subscriber = class {
 	get #totalBufferedBytes() {
 		return this.#bufferedBacklogBytes + this.#pendingBytes;
 	}
-	#pushBacklog(change) {
+	#pushBacklog(entry) {
 		assert(this.#backlog, "cannot push to backlog after catchup completed");
-		this.#backlog.push(change);
-		this.#backlogBytes += change[2].length;
+		this.#backlog.push(entry);
+		this.#backlogBytes += entryBytes(entry);
 		if (this.backlogFull) this.#resolveBacklogFullWaiters();
 	}
 	#resolveBacklogFullWaiters() {
@@ -369,8 +391,8 @@ var Subscriber = class {
 		let inFlightBytes = 0;
 		try {
 			for (;;) {
-				const change = this.#backlog?.shift();
-				if (!change) {
+				const entry = this.#backlog?.shift();
+				if (!entry) {
 					const closed = this.#backlog === null;
 					this.#backlog = null;
 					this.#backlogBytes = 0;
@@ -378,14 +400,17 @@ var Subscriber = class {
 					if (!closed && !this.#aligned) {
 						this.#aligned = true;
 						this.#onAligned?.(this);
+						this.#lagReportingDisabled?.then((disabled) => {
+							if (disabled) this.sendStatus(LAG_REPORTING_DISABLED);
+						});
 					}
 					break;
 				}
-				const bytes = change[2].length;
+				const bytes = entryBytes(entry);
 				this.#backlogBytes -= bytes;
 				this.#backlogInFlightBytes += bytes;
 				this.#backlogBackpressure.releaseIfUnderLowWater(this.#bufferedBacklogBytes);
-				const promise = this.#sendChange(change).finally(() => {
+				const promise = (isQueuedStatus(entry) ? this.#sendStringifiedDownstream(entry.status).then(() => {}) : this.#sendChange(entry)).finally(() => {
 					this.#backlogInFlightBytes -= bytes;
 					this.#backlogBackpressure.releaseIfUnderLowWater(this.#bufferedBacklogBytes);
 				});
@@ -430,6 +455,16 @@ var ByteBackpressureGate = class {
 		for (const waiter of waiters) waiter.resolve();
 	}
 };
+/**
+* Sent to an aligned subscriber when lag reporting is disabled upstream,
+* signaling that the subscriber is caught up. This is a lag report with a
+* `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured
+* (see `isLagReportingDisabledSignal()` in the replicator's recorder).
+*/
+var LAG_REPORTING_DISABLED = {
+	tag: "status",
+	lagReport: { nextSendTimeMs: 0 }
+};
 //#endregion
 export { Subscriber };

package/out/zero-cache/src/services/change-streamer/subscriber.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#latestStatus","#wsBatched","#backlogBackpressure","#backlogFullWaiters","#onAck","#backfills","#onAligned","#onBackfillIgnored","#watermark","#acked","#backlog","#aligned","#bufferedBacklogBytes","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#trackBackfills","#sendPreSerializedDownstream","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#pending","#pendingBytes","#processed","#backlogCount","#samples","#totalBufferedBytes","#missedLastTimeout","#laggingSinceMs","#backlogBytes","#resolveBacklogFullWaiters","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n ChangeTag,\n Downstream,\n Status,\n WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n backlogHighWaterBytes?: number \| undefined;\n backlogLowWaterRatio?: number \| undefined;\n wsBatched?: boolean \| undefined;\n\n /*\n Called whenever the subscriber's acked watermark advances, i.e. when it\n * has confirmed that a commit was durably applied to its replica. The ACK\n * therefore lags the subscriber's replica rather than leading it.\n /\n onAck?: ((watermark: string) => void) \| undefined;\n\n /\n The subscriber's pending backfills (protocol v8+), which are tracked\n * over the changes sent to the subscriber.\n /\n backfills?: BackfillState \| undefined;\n\n /\n Called once the subscriber has been sent all changes up to the head of\n * the stream, i.e. its catchup and backlog have been flushed. From this\n * point on, the subscriber's tracked {@link backfills} reflect the same\n * position in the stream as that of the change-streamer.\n /\n onAligned?: ((subscriber: Subscriber) => void) \| undefined;\n\n /\n Called when a `backfill` or `backfill-completed` message is ignored for\n * some of the subscriber's pending backfill columns, after the subscriber\n * is aligned.\n /\n onBackfillIgnored?: ((columns: string[]) => void) \| undefined;\n};\n\nexport type BacklogFullWait = {\n readonly promise: Promise<void>;\n cancel(): void;\n};\n\nexport type SubscriberStats = {\n processRate: number;\n pending: number;\n backlog: number;\n backlogBytes: number;\n totalBufferedBytes: number;\n missedLastTimeout: boolean;\n};\n\n/\n Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n /\nexport class Subscriber {\n readonly #protocolVersion: number;\n readonly id: string;\n readonly mode: ReplicatorMode;\n readonly #downstream: Subscription<string \| PreSerializedBatch>;\n readonly #latestStatus: () => Status;\n readonly #wsBatched: boolean;\n #watermark: string;\n #acked: string;\n #backlog: RingBuffer<WatermarkedChange> \| null;\n // While catchup is running, live changes are buffered here instead of being\n // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n // shifting an array, which matters when a subscriber is far behind.\n #backlogBytes = 0;\n #backlogInFlightBytes = 0;\n #backlogDrain: Promise<void> \| null = null;\n readonly #backlogBackpressure: ByteBackpressureGate;\n readonly #backlogFullWaiters = new Set<Resolver<void>>();\n readonly #onAck: ((watermark: string) => void) \| undefined;\n readonly #backfills: BackfillState \| undefined;\n readonly #onAligned: ((subscriber: Subscriber) => void) \| undefined;\n readonly #onBackfillIgnored: ((columns: string[]) => void) \| undefined;\n #aligned = false;\n\n constructor(\n protocolVersion: number,\n id: string,\n mode: ReplicatorMode,\n watermark: string,\n downstream: Subscription<string \| PreSerializedBatch>,\n latestStatus: () => Status,\n options: SubscriberOptions = {},\n ) {\n this.#protocolVersion = protocolVersion;\n this.id = id;\n this.mode = mode;\n this.#downstream = downstream;\n this.#latestStatus = latestStatus;\n this.#wsBatched = options.wsBatched ?? false;\n this.#watermark = watermark;\n this.#acked = watermark;\n this.#backlog = new RingBuffer();\n this.#backlogBackpressure = new ByteBackpressureGate(\n options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n );\n this.#onAck = options.onAck;\n this.#backfills = options.backfills;\n this.#onAligned = options.onAligned;\n this.#onBackfillIgnored = options.onBackfillIgnored;\n }\n\n /\n The subscriber's tracked pending backfills, or `undefined` if the\n * subscriber does not report them (i.e. protocol < v8).\n /\n get backfills(): BackfillState \| undefined {\n return this.#backfills;\n }\n\n /\n Whether the subscriber has been sent all changes up to the head of the\n * stream (see {@link SubscriberOptions.onAligned}).\n /\n get aligned(): boolean {\n return this.#aligned;\n }\n\n #trackBackfills(change: WatermarkedChange) {\n if (this.#backfills) {\n const ignored = this.#backfills.applySerialized(change);\n if (ignored.length && this.#aligned) {\n this.#onBackfillIgnored?.(ignored);\n }\n }\n }\n\n get watermark() {\n return this.#watermark;\n }\n\n get acked() {\n return this.#acked;\n }\n\n /\n Whether the backlog of live changes buffered during catchup has reached the\n * point at which {@link send()} stops resolving. Past it the subscriber is no\n * longer free: it holds up every subsequent flush, and with no other\n * subscriber to form a majority it stalls replication outright.\n /\n get backlogFull() {\n return (\n this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n );\n }\n\n /\n @returns whether the subscriber is currently sending backlogged messages,\n * vs caught up and sending the \"head\" of the replication stream.\n /\n isBacklogged() {\n return this.#backlog !== null \|\| this.#bufferedBacklogBytes > 0;\n }\n\n /\n Resolves the first time {@link backlogFull} becomes true, so that a caller\n * holding the subscriber in catchup can give up at the moment the subscriber\n * starts costing replication rather than on a timer. Call cancel() when the\n * wait is no longer needed so the subscriber does not retain it.\n /\n whenBacklogFull(): BacklogFullWait {\n if (this.backlogFull) {\n return {promise: promiseVoid, cancel() {}};\n }\n const r = resolver<void>();\n this.#backlogFullWaiters.add(r);\n return {\n promise: r.promise,\n cancel: () => {\n this.#backlogFullWaiters.delete(r);\n },\n };\n }\n\n send(change: WatermarkedChange): Promise<void> {\n const [watermark] = change;\n if (watermark > this.#watermark) {\n if (this.#backlog) {\n // During catchup, buffer live changes behind the durable catchup stream.\n // The returned promise applies backpressure if the buffered bytes cross\n // the high water mark.\n this.#pushBacklog(change);\n return this.#maybeWaitForBacklogSpace();\n }\n return this.#sendChange(change);\n }\n return promiseVoid;\n }\n\n sendBatch(\n changes: readonly WatermarkedChange[],\n preSerialized?: PreSerializedBatch \| undefined,\n ): Promise<void> {\n if (changes.length === 0) {\n return promiseVoid;\n }\n\n if (\n this.#wsBatched &&\n preSerialized &&\n !this.#backlog &&\n this.#initialized &&\n changes[0][0] > this.#watermark &&\n (this.#protocolVersion >= 5 \|\|\n changes.every(c => this.supportsMessage(c[1])))\n ) {\n let commitWatermark: string \| undefined;\n for (let i = changes.length - 1; i >= 0; i--) {\n if (changes[i][1] === 'commit') {\n commitWatermark = changes[i][0];\n break;\n }\n }\n if (commitWatermark) {\n this.#watermark = commitWatermark;\n }\n for (const change of changes) {\n this.#trackBackfills(change);\n }\n return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n }\n\n const promises: Promise<void>[] = [];\n for (const change of changes) {\n const p = this.send(change);\n if (p !== promiseVoid) {\n promises.push(p);\n }\n }\n if (promises.length === 0) {\n return promiseVoid;\n }\n if (promises.length === 1) {\n return promises[0];\n }\n return Promise.all(promises).then(() => {});\n }\n\n #initialized = false;\n\n /\n Called once the subscriber's watermark has been validated in the initial\n * catchup process.\n /\n #initialize() {\n if (!this.#initialized) {\n this.#initialized = true;\n this.sendStatus(this.#latestStatus());\n }\n }\n\n sendStatus(status: Status) {\n if (this.#initialized) {\n void this.#sendDownstream(['status', status]);\n }\n }\n\n /* catchup() is called on ChangeEntries loaded from the store. /\n async catchup(change: WatermarkedChange) {\n this.#initialize();\n await this.#sendChange(change);\n }\n\n /\n Marks the Subscribe as \"caught up\" and flushes any backlog of\n * entries that were received during the catchup.\n /\n setCaughtUp(): Promise<void> {\n this.#initialize();\n if (!this.#backlog) {\n return this.#backlogDrain ?? promiseVoid;\n }\n if (!this.#backlogDrain) {\n // Keep #backlog non-null while queued entries are being handed to\n // downstream. That preserves ordering for sends that race with\n // setCaughtUp(): they append to the same backlog instead of bypassing\n // older buffered changes.\n this.#backlogDrain = this.#drainBacklog();\n void this.#backlogDrain.catch(e => this.fail(e));\n }\n return this.#backlogDrain;\n }\n\n async #sendChange(change: WatermarkedChange) {\n const [watermark, tag, json] = change;\n if (watermark <= this.watermark) {\n return;\n }\n if (!this.supportsMessage(tag)) {\n return;\n }\n if (tag === 'commit') {\n this.#watermark = watermark;\n }\n this.#trackBackfills(change);\n const result = await this.#sendStringifiedDownstream(json);\n if (tag === 'commit' && result === 'consumed') {\n // Sends can complete out of order (e.g. the bounded window in\n // #drainBacklog), so the ack only advances monotonically, and listeners\n // are only notified when it does.\n const acked = max(this.#acked, watermark);\n if (acked !== this.#acked) {\n this.#acked = acked;\n this.#onAck?.(acked);\n }\n }\n }\n\n #sendDownstream(downstream: Downstream) {\n return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n }\n\n async #sendStringifiedDownstream(json: string) {\n const size = json.length;\n this.#pending++;\n this.#pendingBytes += size;\n const {result} = this.#downstream.push(json);\n try {\n return await result;\n } finally {\n this.#pending--;\n this.#pendingBytes -= size;\n this.#processed++;\n }\n }\n\n async #sendPreSerializedDownstream(\n batch: PreSerializedBatch,\n commitWatermark?: string \| undefined,\n ): Promise<void> {\n const size = batch.byteLength;\n this.#pending += batch.changes.length;\n this.#pendingBytes += size;\n const {result} = this.#downstream.push(batch);\n try {\n const outcome = await result;\n if (commitWatermark && outcome === 'consumed') {\n const acked = max(this.#acked, commitWatermark);\n if (acked !== this.#acked) {\n this.#acked = acked;\n this.#onAck?.(acked);\n }\n }\n } finally {\n this.#pending -= batch.changes.length;\n this.#pendingBytes -= size;\n this.#processed += batch.changes.length;\n }\n }\n\n // `pending` and `processed` stats are tracked by periodically sampling\n // the running totals (by the progress tracker in the Forwarder).\n // This information was originally collected for use in flow control\n // decisions. The final flow control algorithm ended up being simpler\n // than expected and does not actually use this information. However, the\n // stats are still tracked and logged during flow control decisions for\n // debugging, forensics, and potential improvements to the algorithm.\n\n #pending = 0;\n #pendingBytes = 0;\n #processed = 0;\n #samples: {processed: number; timestamp: number}[] = [\n {processed: 0, timestamp: performance.now()},\n ];\n\n /\n The number of downstream messages that have yet to be acked.\n /\n get numPending() {\n return this.#pending + this.#backlogCount;\n }\n\n /\n The total number of downstream messages that the subscriber has\n * processed (i.e. acked).\n /\n get numProcessed() {\n return this.#processed;\n }\n\n /\n Records a new history entry for the number of messages processed,\n * keeping the number of samples bounded to `maxSamples`.\n /\n sampleProcessRate(now: number, maxSamples = 10): this {\n while (this.#samples.length >= maxSamples) {\n this.#samples.shift();\n }\n this.#samples.push({processed: this.#processed, timestamp: now});\n return this;\n }\n\n getStats(): SubscriberStats {\n const pending = this.numPending;\n if (this.#samples.length < 2) {\n return {\n processRate: 0,\n pending,\n backlog: this.#backlogCount,\n backlogBytes: this.#bufferedBacklogBytes,\n totalBufferedBytes: this.#totalBufferedBytes,\n missedLastTimeout: this.#missedLastTimeout,\n };\n }\n const from = this.#samples[0];\n const to = must(this.#samples.at(-1));\n const processed = to.processed - from.processed;\n const seconds = (to.timestamp - from.timestamp) / 1000;\n const processRate = seconds === 0 ? 0 : processed / seconds;\n return {\n processRate,\n pending,\n backlog: this.#backlogCount,\n backlogBytes: this.#bufferedBacklogBytes,\n totalBufferedBytes: this.#totalBufferedBytes,\n missedLastTimeout: this.#missedLastTimeout,\n };\n }\n\n #missedLastTimeout = false;\n #laggingSinceMs: number \| undefined;\n\n trackResponseResult(result: 'on-time' \| 'timed-out') {\n this.#missedLastTimeout = result === 'timed-out';\n if (result === 'on-time') {\n this.#laggingSinceMs = undefined;\n }\n }\n\n /\n Reports the change rate of a slow subscriber (i.e. that missed the last\n * timeout) compared to the change rate of the (slowest) subscriber that\n * responded on time.\n \n * `lagging` indicates that this subscriber is slower\n * * `catching-up` indicates that it is faster\n \n Returns the total duration in which subscriber has been continuously\n * reported as `lagging`.\n /\n reportChangeRate(now: number, status: 'lagging' \| 'catching-up') {\n assert(\n this.#missedLastTimeout,\n `reportChangeRate should only be called for slow subscribers`,\n );\n if (status === 'catching-up') {\n this.#laggingSinceMs = undefined;\n return 0;\n }\n this.#laggingSinceMs ??= now;\n return now - this.#laggingSinceMs;\n }\n\n supportsMessage(tag: ChangeTag) {\n switch (tag) {\n case 'update-table-metadata':\n // update-table-row-key is only understood by subscribers >= protocol v5\n return this.#protocolVersion >= 5;\n }\n return true;\n }\n\n /\n Ends the subscription without sending a downstream `['error', ...]`.\n \n This is deliberate, and not the same as reporting the failure to the\n * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n * shuts down to restore a fresh replica from litestream, whereas a clean end\n * backs off and re-subscribes. The failures routed here -- storer catchup\n * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n * reconnect is the proportionate response and a fleet-wide restore is not.\n \n Callers are responsible for logging `err`; it is not carried downstream.\n /\n fail(_err?: unknown) {\n this.close();\n }\n\n close(error?: ErrorType, message?: string) {\n // Closing the subscriber must also release producers that are blocked on\n // backlog capacity; there is no future drain that could wake them.\n this.#backlog = null;\n this.#backlogBytes = 0;\n this.#backlogBackpressure.releaseAll();\n // The backlog can no longer grow, so nothing would ever resolve these.\n // Waiters re-check backlogFull, which is now false, and see the close.\n this.#resolveBacklogFullWaiters();\n\n if (error !== undefined) {\n // Wait for the ACK of the error message before closing the connection.\n void this.#sendDownstream(['error', {type: error, message}]).finally(() =>\n this.#downstream.cancel(),\n );\n } else {\n this.#downstream.cancel();\n }\n }\n\n get #backlogCount() {\n return this.#backlog?.size ?? 0;\n }\n\n get #bufferedBacklogBytes() {\n // Include entries already handed to downstream but not yet consumed. Without\n // this, setCaughtUp() could move bytes out of #backlog faster than the\n // downstream Subscription can process them and release producers too early.\n return this.#backlogBytes + this.#backlogInFlightBytes;\n }\n\n get #totalBufferedBytes() {\n return this.#bufferedBacklogBytes + this.#pendingBytes;\n }\n\n #pushBacklog(change: WatermarkedChange) {\n assert(this.#backlog, 'cannot push to backlog after catchup completed');\n this.#backlog.push(change);\n this.#backlogBytes += change[2].length;\n if (this.backlogFull) {\n this.#resolveBacklogFullWaiters();\n }\n }\n\n #resolveBacklogFullWaiters() {\n const waiters = [...this.#backlogFullWaiters];\n this.#backlogFullWaiters.clear();\n for (const waiter of waiters) {\n waiter.resolve();\n }\n }\n\n #maybeWaitForBacklogSpace(): Promise<void> {\n return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n }\n\n async #drainBacklog() {\n const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n let inFlightBytes = 0;\n\n try {\n for (;;) {\n const change = this.#backlog?.shift();\n if (!change) {\n const closed = this.#backlog === null;\n this.#backlog = null;\n this.#backlogBytes = 0;\n this.#backlogBackpressure.releaseIfUnderLowWater(\n this.#bufferedBacklogBytes,\n );\n if (!closed && !this.#aligned) {\n // All changes up to the head have been handed to #sendChange()\n // (and thus tracked), and subsequent changes are sent directly.\n this.#aligned = true;\n this.#onAligned?.(this);\n }\n break;\n }\n\n const bytes = change[2].length;\n this.#backlogBytes -= bytes;\n this.#backlogInFlightBytes += bytes;\n this.#backlogBackpressure.releaseIfUnderLowWater(\n this.#bufferedBacklogBytes,\n );\n\n // Send backlog entries in order, but keep only a bounded byte window in\n // flight. This avoids replacing one unbounded buffer with another inside\n // the downstream Subscription during catchup completion.\n const promise = this.#sendChange(change).finally(() => {\n this.#backlogInFlightBytes -= bytes;\n this.#backlogBackpressure.releaseIfUnderLowWater(\n this.#bufferedBacklogBytes,\n );\n });\n inFlight.push({promise, bytes});\n inFlightBytes += bytes;\n\n while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n const next = must(inFlight.shift());\n await next.promise;\n inFlightBytes -= next.bytes;\n }\n }\n\n for (const {promise} of inFlight) {\n await promise;\n }\n } finally {\n this.#backlogDrain = null;\n this.#backlogBackpressure.releaseIfUnderLowWater(\n this.#bufferedBacklogBytes,\n );\n }\n }\n}\n\nclass ByteBackpressureGate {\n readonly highWaterBytes: number;\n readonly #lowWaterBytes: number;\n readonly #waiters: Resolver<void>[] = [];\n\n constructor(highWaterBytes: number, lowWaterRatio: number) {\n this.highWaterBytes = Math.max(1, highWaterBytes);\n this.#lowWaterBytes =\n this.highWaterBytes Math.min(1, Math.max(0, lowWaterRatio));\n }\n\n waitForSpace(bufferedBytes: number): Promise<void> {\n if (bufferedBytes < this.highWaterBytes) {\n return promiseVoid;\n }\n\n // One waiter represents one send() call that has already appended its\n // change. The producer is released when the backlog falls back below the low\n // water mark or the subscriber closes.\n const r = resolver<void>();\n this.#waiters.push(r);\n return r.promise;\n }\n\n releaseIfUnderLowWater(bufferedBytes: number) {\n if (this.#waiters.length === 0 \|\| bufferedBytes > this.#lowWaterBytes) {\n return;\n }\n\n // Use a low water mark so waiting producers are released in batches instead\n // of waking one at a time around the high water boundary.\n this.releaseAll();\n }\n\n releaseAll() {\n const waiters = this.#waiters.splice(0);\n for (const waiter of waiters) {\n waiter.resolve();\n }\n }\n}\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;;;;;;;;AAyDxC,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA,WAAW;CAEX,YACE,iBACA,IACA,MACA,WACA,YACA,cACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAKC,cAAc;EACnB,KAAKC,gBAAgB;EACrB,KAAKC,aAAa,QAAQ,aAAa;EACvC,KAAKO,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKR,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAKE,SAAS,QAAQ;EACtB,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,qBAAqB,QAAQ;CACpC;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAKF;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAKM;CACd;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKN,YAAY;GACnB,MAAM,UAAU,KAAKA,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAKM,UACzB,KAAKJ,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAKC;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAKG,yBAAyB,KAAKV,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAKQ,aAAa,QAAQ,KAAKE,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAKT,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAKA,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKK,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKG,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAKd,cACL,iBACA,CAAC,KAAKS,YACN,KAAKM,gBACL,QAAQ,GAAG,KAAK,KAAKR,eACpB,KAAKV,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,GAAG,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,GAAG;IAC7B;GACF;GAEF,IAAI,iBACF,KAAKU,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAKS,gBAAgB,MAAM;GAE7B,OAAO,KAAKC,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,EAAE,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKF,cAAc;GACtB,KAAKA,eAAe;GACpB,KAAK,WAAW,KAAKhB,cAAc,CAAC;EACtC;CACF;CAEA,WAAW,QAAgB;EACzB,IAAI,KAAKgB,cACP,KAAUG,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAEhD;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKL,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKK,YAAY;EACjB,IAAI,CAAC,KAAKV,UACR,OAAO,KAAKW,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAMN,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKP,aAAa;EAEpB,KAAKS,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAKM,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YAAY;GAI7C,MAAM,QAAQ,IAAI,KAAKd,QAAQ,SAAS;GACxC,IAAI,UAAU,KAAKA,QAAQ;IACzB,KAAKA,SAAS;IACd,KAAKL,SAAS,KAAK;GACrB;EACF;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKmB,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAKC;EACL,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK1B,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAKyB;GACL,KAAKC,iBAAiB;GACtB,KAAKC;EACP;CACF;CAEA,MAAMR,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAKM,YAAY,MAAM,QAAQ;EAC/B,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK1B,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YAAY;IAC7C,MAAM,QAAQ,IAAI,KAAKU,QAAQ,eAAe;IAC9C,IAAI,UAAU,KAAKA,QAAQ;KACzB,KAAKA,SAAS;KACd,KAAKL,SAAS,KAAK;IACrB;GACF;EACF,UAAU;GACR,KAAKoB,YAAY,MAAM,QAAQ;GAC/B,KAAKC,iBAAiB;GACtB,KAAKC,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKF,WAAW,KAAKG;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKf;GACnB,oBAAoB,KAAKiB;GACzB,mBAAmB,KAAKC;EAC1B;EAEF,MAAM,OAAO,KAAKF,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKf;GACnB,oBAAoB,KAAKiB;GACzB,mBAAmB,KAAKC;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAKA,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAKC,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAKD,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAKC,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAKA,oBAAoB;EACzB,OAAO,MAAM,KAAKA;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAKjC,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MAAM,OAAmB,SAAkB;EAGzC,KAAKY,WAAW;EAChB,KAAKsB,gBAAgB;EACrB,KAAK9B,qBAAqB,WAAW;EAGrC,KAAK+B,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAUd,gBAAgB,CAAC,SAAS;GAAC,MAAM;GAAO;EAAO,CAAC,CAAC,EAAE,cAC3D,KAAKpB,YAAY,OAAO,CAC1B;OAEA,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAI4B,gBAAgB;EAClB,OAAO,KAAKjB,UAAU,QAAQ;CAChC;CAEA,IAAIE,wBAAwB;EAI1B,OAAO,KAAKoB,gBAAgB,KAAKE;CACnC;CAEA,IAAIL,sBAAsB;EACxB,OAAO,KAAKjB,wBAAwB,KAAKa;CAC3C;CAEA,aAAa,QAA2B;EACtC,OAAO,KAAKf,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,MAAM;EACzB,KAAKsB,iBAAiB,OAAO,GAAG;EAChC,IAAI,KAAK,aACP,KAAKC,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAK9B,mBAAmB;EAC5C,KAAKA,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAKD,qBAAqB,aAAa,KAAKU,qBAAqB;CAC1E;CAEA,MAAMU,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,SAAS,KAAKZ,UAAU,MAAM;IACpC,IAAI,CAAC,QAAQ;KACX,MAAM,SAAS,KAAKA,aAAa;KACjC,KAAKA,WAAW;KAChB,KAAKsB,gBAAgB;KACrB,KAAK9B,qBAAqB,uBACxB,KAAKU,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAKD,UAAU;MAG7B,KAAKA,WAAW;MAChB,KAAKL,aAAa,IAAI;KACxB;KACA;IACF;IAEA,MAAM,QAAQ,OAAO,GAAG;IACxB,KAAK0B,iBAAiB;IACtB,KAAKE,yBAAyB;IAC9B,KAAKhC,qBAAqB,uBACxB,KAAKU,qBACP;IAKA,MAAM,UAAU,KAAKG,YAAY,MAAM,EAAE,cAAc;KACrD,KAAKmB,yBAAyB;KAC9B,KAAKhC,qBAAqB,uBACxB,KAAKU,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKV,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKmB,gBAAgB;GACrB,KAAKnB,qBAAqB,uBACxB,KAAKU,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAKuB,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF"}
1	+ {"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#wsBatched","#backlogBackpressure","#backlogFullWaiters","#onAck","#backfills","#onAligned","#onBackfillIgnored","#lagReportingDisabled","#watermark","#acked","#backlog","#aligned","#bufferedBacklogBytes","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#trackBackfills","#sendPreSerializedDownstream","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#pending","#pendingBytes","#processed","#backlogCount","#samples","#totalBufferedBytes","#missedLastTimeout","#laggingSinceMs","#backlogBytes","#resolveBacklogFullWaiters","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n ChangeTag,\n Downstream,\n Status,\n WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n backlogHighWaterBytes?: number \| undefined;\n backlogLowWaterRatio?: number \| undefined;\n wsBatched?: boolean \| undefined;\n\n /*\n Called whenever the subscriber's acked watermark advances, i.e. when it\n * has confirmed that a commit was durably applied to its replica. The ACK\n * therefore lags the subscriber's replica rather than leading it.\n /\n onAck?: ((watermark: string) => void) \| undefined;\n\n /\n The subscriber's pending backfills (protocol v8+), which are tracked\n * over the changes sent to the subscriber.\n /\n backfills?: BackfillState \| undefined;\n\n /\n Called once the subscriber has been sent all changes up to the head of\n * the stream, i.e. its catchup and backlog have been flushed. From this\n * point on, the subscriber's tracked {@link backfills} reflect the same\n * position in the stream as that of the change-streamer.\n /\n onAligned?: ((subscriber: Subscriber) => void) \| undefined;\n\n /\n Called when a `backfill` or `backfill-completed` message is ignored for\n * some of the subscriber's pending backfill columns, after the subscriber\n * is aligned.\n /\n onBackfillIgnored?: ((columns: string[]) => void) \| undefined;\n\n /\n Resolves to whether lag reporting is disabled upstream. If so, a\n * {@link LAG_REPORTING_DISABLED} status is sent once the subscriber is\n * aligned, as a stand-in for the lag reports that a subscriber would\n * otherwise use to determine that it is caught up (e.g. to advertise\n * readiness to serve traffic).\n /\n lagReportingDisabled?: Promise<boolean> \| undefined;\n};\n\n/\n A status message queued in the backlog, so that it is sent downstream in\n * the order in which it was received relative to (backlogged) changes.\n /\ntype QueuedStatus = {readonly status: string};\n\ntype BacklogEntry = WatermarkedChange \| QueuedStatus;\n\nfunction isQueuedStatus(entry: BacklogEntry): entry is QueuedStatus {\n return !Array.isArray(entry);\n}\n\nfunction entryBytes(entry: BacklogEntry): number {\n return isQueuedStatus(entry) ? entry.status.length : entry[2].length;\n}\n\nexport type BacklogFullWait = {\n readonly promise: Promise<void>;\n cancel(): void;\n};\n\nexport type SubscriberStats = {\n processRate: number;\n pending: number;\n backlog: number;\n backlogBytes: number;\n totalBufferedBytes: number;\n missedLastTimeout: boolean;\n};\n\n/\n Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n /\nexport class Subscriber {\n readonly #protocolVersion: number;\n readonly id: string;\n readonly mode: ReplicatorMode;\n readonly #downstream: Subscription<string \| PreSerializedBatch>;\n readonly #wsBatched: boolean;\n #watermark: string;\n #acked: string;\n #backlog: RingBuffer<BacklogEntry> \| null;\n // While catchup is running, live changes are buffered here instead of being\n // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n // shifting an array, which matters when a subscriber is far behind.\n #backlogBytes = 0;\n #backlogInFlightBytes = 0;\n #backlogDrain: Promise<void> \| null = null;\n readonly #backlogBackpressure: ByteBackpressureGate;\n readonly #backlogFullWaiters = new Set<Resolver<void>>();\n readonly #onAck: ((watermark: string) => void) \| undefined;\n readonly #backfills: BackfillState \| undefined;\n readonly #onAligned: ((subscriber: Subscriber) => void) \| undefined;\n readonly #onBackfillIgnored: ((columns: string[]) => void) \| undefined;\n readonly #lagReportingDisabled: Promise<boolean> \| undefined;\n #aligned = false;\n\n constructor(\n protocolVersion: number,\n id: string,\n mode: ReplicatorMode,\n watermark: string,\n downstream: Subscription<string \| PreSerializedBatch>,\n options: SubscriberOptions = {},\n ) {\n this.#protocolVersion = protocolVersion;\n this.id = id;\n this.mode = mode;\n this.#downstream = downstream;\n this.#wsBatched = options.wsBatched ?? false;\n this.#watermark = watermark;\n this.#acked = watermark;\n this.#backlog = new RingBuffer();\n this.#backlogBackpressure = new ByteBackpressureGate(\n options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n );\n this.#onAck = options.onAck;\n this.#backfills = options.backfills;\n this.#onAligned = options.onAligned;\n this.#onBackfillIgnored = options.onBackfillIgnored;\n this.#lagReportingDisabled = options.lagReportingDisabled;\n }\n\n /\n The subscriber's tracked pending backfills, or `undefined` if the\n * subscriber does not report them (i.e. protocol < v8).\n /\n get backfills(): BackfillState \| undefined {\n return this.#backfills;\n }\n\n /\n Whether the subscriber has been sent all changes up to the head of the\n * stream (see {@link SubscriberOptions.onAligned}).\n /\n get aligned(): boolean {\n return this.#aligned;\n }\n\n #trackBackfills(change: WatermarkedChange) {\n if (this.#backfills) {\n const ignored = this.#backfills.applySerialized(change);\n if (ignored.length && this.#aligned) {\n this.#onBackfillIgnored?.(ignored);\n }\n }\n }\n\n get watermark() {\n return this.#watermark;\n }\n\n get acked() {\n return this.#acked;\n }\n\n /\n Whether the backlog of live changes buffered during catchup has reached the\n * point at which {@link send()} stops resolving. Past it the subscriber is no\n * longer free: it holds up every subsequent flush, and with no other\n * subscriber to form a majority it stalls replication outright.\n /\n get backlogFull() {\n return (\n this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n );\n }\n\n /\n @returns whether the subscriber is currently sending backlogged messages,\n * vs caught up and sending the \"head\" of the replication stream.\n /\n isBacklogged() {\n return this.#backlog !== null \|\| this.#bufferedBacklogBytes > 0;\n }\n\n /\n Resolves the first time {@link backlogFull} becomes true, so that a caller\n * holding the subscriber in catchup can give up at the moment the subscriber\n * starts costing replication rather than on a timer. Call cancel() when the\n * wait is no longer needed so the subscriber does not retain it.\n /\n whenBacklogFull(): BacklogFullWait {\n if (this.backlogFull) {\n return {promise: promiseVoid, cancel() {}};\n }\n const r = resolver<void>();\n this.#backlogFullWaiters.add(r);\n return {\n promise: r.promise,\n cancel: () => {\n this.#backlogFullWaiters.delete(r);\n },\n };\n }\n\n send(change: WatermarkedChange): Promise<void> {\n const [watermark] = change;\n if (watermark > this.#watermark) {\n if (this.#backlog) {\n // During catchup, buffer live changes behind the durable catchup stream.\n // The returned promise applies backpressure if the buffered bytes cross\n // the high water mark.\n this.#pushBacklog(change);\n return this.#maybeWaitForBacklogSpace();\n }\n return this.#sendChange(change);\n }\n return promiseVoid;\n }\n\n sendBatch(\n changes: readonly WatermarkedChange[],\n preSerialized?: PreSerializedBatch \| undefined,\n ): Promise<void> {\n if (changes.length === 0) {\n return promiseVoid;\n }\n\n if (\n this.#wsBatched &&\n preSerialized &&\n !this.#backlog &&\n this.#initialized &&\n changes[0][0] > this.#watermark &&\n (this.#protocolVersion >= 5 \|\|\n changes.every(c => this.supportsMessage(c[1])))\n ) {\n let commitWatermark: string \| undefined;\n for (let i = changes.length - 1; i >= 0; i--) {\n if (changes[i][1] === 'commit') {\n commitWatermark = changes[i][0];\n break;\n }\n }\n if (commitWatermark) {\n this.#watermark = commitWatermark;\n }\n for (const change of changes) {\n this.#trackBackfills(change);\n }\n return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n }\n\n const promises: Promise<void>[] = [];\n for (const change of changes) {\n const p = this.send(change);\n if (p !== promiseVoid) {\n promises.push(p);\n }\n }\n if (promises.length === 0) {\n return promiseVoid;\n }\n if (promises.length === 1) {\n return promises[0];\n }\n return Promise.all(promises).then(() => {});\n }\n\n #initialized = false;\n\n /\n Called once the subscriber's watermark has been validated in the initial\n * catchup process.\n /\n #initialize() {\n if (!this.#initialized) {\n this.#initialized = true;\n // The initial status precedes the catchup, so it bypasses the backlog.\n void this.#sendDownstream(['status', {tag: 'status'}]);\n }\n }\n\n /\n Sends a status message (e.g. a lag report) downstream. While the\n * subscriber is catching up, the status is queued in the backlog behind\n * the changes that preceded it in the replication stream, so that the\n * subscriber processes it in stream order. This is necessary for lag\n * reports to measure the lag of the subscriber, including its catchup.\n \n Status messages received before the subscriber is initialized are\n * dropped.\n /\n sendStatus(status: Status) {\n if (!this.#initialized) {\n return;\n }\n const downstream: Downstream = ['status', status];\n if (this.#backlog) {\n this.#pushBacklog({status: BigIntJSON.stringify(downstream)});\n return;\n }\n void this.#sendDownstream(downstream);\n }\n\n /* catchup() is called on ChangeEntries loaded from the store. /\n async catchup(change: WatermarkedChange) {\n this.#initialize();\n await this.#sendChange(change);\n }\n\n /\n Marks the Subscribe as \"caught up\" and flushes any backlog of\n * entries that were received during the catchup.\n /\n setCaughtUp(): Promise<void> {\n this.#initialize();\n if (!this.#backlog) {\n return this.#backlogDrain ?? promiseVoid;\n }\n if (!this.#backlogDrain) {\n // Keep #backlog non-null while queued entries are being handed to\n // downstream. That preserves ordering for sends that race with\n // setCaughtUp(): they append to the same backlog instead of bypassing\n // older buffered changes.\n this.#backlogDrain = this.#drainBacklog();\n void this.#backlogDrain.catch(e => this.fail(e));\n }\n return this.#backlogDrain;\n }\n\n async #sendChange(change: WatermarkedChange) {\n const [watermark, tag, json] = change;\n if (watermark <= this.watermark) {\n return;\n }\n if (!this.supportsMessage(tag)) {\n return;\n }\n if (tag === 'commit') {\n this.#watermark = watermark;\n }\n this.#trackBackfills(change);\n const result = await this.#sendStringifiedDownstream(json);\n if (tag === 'commit' && result === 'consumed') {\n // Sends can complete out of order (e.g. the bounded window in\n // #drainBacklog), so the ack only advances monotonically, and listeners\n // are only notified when it does.\n const acked = max(this.#acked, watermark);\n if (acked !== this.#acked) {\n this.#acked = acked;\n this.#onAck?.(acked);\n }\n }\n }\n\n #sendDownstream(downstream: Downstream) {\n return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n }\n\n async #sendStringifiedDownstream(json: string) {\n const size = json.length;\n this.#pending++;\n this.#pendingBytes += size;\n const {result} = this.#downstream.push(json);\n try {\n return await result;\n } finally {\n this.#pending--;\n this.#pendingBytes -= size;\n this.#processed++;\n }\n }\n\n async #sendPreSerializedDownstream(\n batch: PreSerializedBatch,\n commitWatermark?: string \| undefined,\n ): Promise<void> {\n const size = batch.byteLength;\n this.#pending += batch.changes.length;\n this.#pendingBytes += size;\n const {result} = this.#downstream.push(batch);\n try {\n const outcome = await result;\n if (commitWatermark && outcome === 'consumed') {\n const acked = max(this.#acked, commitWatermark);\n if (acked !== this.#acked) {\n this.#acked = acked;\n this.#onAck?.(acked);\n }\n }\n } finally {\n this.#pending -= batch.changes.length;\n this.#pendingBytes -= size;\n this.#processed += batch.changes.length;\n }\n }\n\n // `pending` and `processed` stats are tracked by periodically sampling\n // the running totals (by the progress tracker in the Forwarder).\n // This information was originally collected for use in flow control\n // decisions. The final flow control algorithm ended up being simpler\n // than expected and does not actually use this information. However, the\n // stats are still tracked and logged during flow control decisions for\n // debugging, forensics, and potential improvements to the algorithm.\n\n #pending = 0;\n #pendingBytes = 0;\n #processed = 0;\n #samples: {processed: number; timestamp: number}[] = [\n {processed: 0, timestamp: performance.now()},\n ];\n\n /\n The number of downstream messages that have yet to be acked.\n /\n get numPending() {\n return this.#pending + this.#backlogCount;\n }\n\n /\n The total number of downstream messages that the subscriber has\n * processed (i.e. acked).\n /\n get numProcessed() {\n return this.#processed;\n }\n\n /\n Records a new history entry for the number of messages processed,\n * keeping the number of samples bounded to `maxSamples`.\n /\n sampleProcessRate(now: number, maxSamples = 10): this {\n while (this.#samples.length >= maxSamples) {\n this.#samples.shift();\n }\n this.#samples.push({processed: this.#processed, timestamp: now});\n return this;\n }\n\n getStats(): SubscriberStats {\n const pending = this.numPending;\n if (this.#samples.length < 2) {\n return {\n processRate: 0,\n pending,\n backlog: this.#backlogCount,\n backlogBytes: this.#bufferedBacklogBytes,\n totalBufferedBytes: this.#totalBufferedBytes,\n missedLastTimeout: this.#missedLastTimeout,\n };\n }\n const from = this.#samples[0];\n const to = must(this.#samples.at(-1));\n const processed = to.processed - from.processed;\n const seconds = (to.timestamp - from.timestamp) / 1000;\n const processRate = seconds === 0 ? 0 : processed / seconds;\n return {\n processRate,\n pending,\n backlog: this.#backlogCount,\n backlogBytes: this.#bufferedBacklogBytes,\n totalBufferedBytes: this.#totalBufferedBytes,\n missedLastTimeout: this.#missedLastTimeout,\n };\n }\n\n #missedLastTimeout = false;\n #laggingSinceMs: number \| undefined;\n\n trackResponseResult(result: 'on-time' \| 'timed-out') {\n this.#missedLastTimeout = result === 'timed-out';\n if (result === 'on-time') {\n this.#laggingSinceMs = undefined;\n }\n }\n\n /\n Reports the change rate of a slow subscriber (i.e. that missed the last\n * timeout) compared to the change rate of the (slowest) subscriber that\n * responded on time.\n \n * `lagging` indicates that this subscriber is slower\n * * `catching-up` indicates that it is faster\n \n Returns the total duration in which subscriber has been continuously\n * reported as `lagging`.\n /\n reportChangeRate(now: number, status: 'lagging' \| 'catching-up') {\n assert(\n this.#missedLastTimeout,\n `reportChangeRate should only be called for slow subscribers`,\n );\n if (status === 'catching-up') {\n this.#laggingSinceMs = undefined;\n return 0;\n }\n this.#laggingSinceMs ??= now;\n return now - this.#laggingSinceMs;\n }\n\n supportsMessage(tag: ChangeTag) {\n switch (tag) {\n case 'update-table-metadata':\n // update-table-row-key is only understood by subscribers >= protocol v5\n return this.#protocolVersion >= 5;\n }\n return true;\n }\n\n /\n Ends the subscription without sending a downstream `['error', ...]`.\n \n This is deliberate, and not the same as reporting the failure to the\n * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n * shuts down to restore a fresh replica from litestream, whereas a clean end\n * backs off and re-subscribes. The failures routed here -- storer catchup\n * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n * reconnect is the proportionate response and a fleet-wide restore is not.\n \n Callers are responsible for logging `err`; it is not carried downstream.\n /\n fail(_err?: unknown) {\n this.close();\n }\n\n close(error?: ErrorType, message?: string) {\n // Closing the subscriber must also release producers that are blocked on\n // backlog capacity; there is no future drain that could wake them.\n this.#backlog = null;\n this.#backlogBytes = 0;\n this.#backlogBackpressure.releaseAll();\n // The backlog can no longer grow, so nothing would ever resolve these.\n // Waiters re-check backlogFull, which is now false, and see the close.\n this.#resolveBacklogFullWaiters();\n\n if (error !== undefined) {\n // Wait for the ACK of the error message before closing the connection.\n void this.#sendDownstream(['error', {type: error, message}]).finally(() =>\n this.#downstream.cancel(),\n );\n } else {\n this.#downstream.cancel();\n }\n }\n\n get #backlogCount() {\n return this.#backlog?.size ?? 0;\n }\n\n get #bufferedBacklogBytes() {\n // Include entries already handed to downstream but not yet consumed. Without\n // this, setCaughtUp() could move bytes out of #backlog faster than the\n // downstream Subscription can process them and release producers too early.\n return this.#backlogBytes + this.#backlogInFlightBytes;\n }\n\n get #totalBufferedBytes() {\n return this.#bufferedBacklogBytes + this.#pendingBytes;\n }\n\n #pushBacklog(entry: BacklogEntry) {\n assert(this.#backlog, 'cannot push to backlog after catchup completed');\n this.#backlog.push(entry);\n this.#backlogBytes += entryBytes(entry);\n if (this.backlogFull) {\n this.#resolveBacklogFullWaiters();\n }\n }\n\n #resolveBacklogFullWaiters() {\n const waiters = [...this.#backlogFullWaiters];\n this.#backlogFullWaiters.clear();\n for (const waiter of waiters) {\n waiter.resolve();\n }\n }\n\n #maybeWaitForBacklogSpace(): Promise<void> {\n return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n }\n\n async #drainBacklog() {\n const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n let inFlightBytes = 0;\n\n try {\n for (;;) {\n const entry = this.#backlog?.shift();\n if (!entry) {\n const closed = this.#backlog === null;\n this.#backlog = null;\n this.#backlogBytes = 0;\n this.#backlogBackpressure.releaseIfUnderLowWater(\n this.#bufferedBacklogBytes,\n );\n if (!closed && !this.#aligned) {\n // All changes up to the head have been handed to #sendChange()\n // (and thus tracked), and subsequent changes are sent directly.\n this.#aligned = true;\n this.#onAligned?.(this);\n void this.#lagReportingDisabled?.then(disabled => {\n if (disabled) {\n this.sendStatus(LAG_REPORTING_DISABLED);\n }\n });\n }\n break;\n }\n\n const bytes = entryBytes(entry);\n this.#backlogBytes -= bytes;\n this.#backlogInFlightBytes += bytes;\n this.#backlogBackpressure.releaseIfUnderLowWater(\n this.#bufferedBacklogBytes,\n );\n\n // Send backlog entries in order, but keep only a bounded byte window in\n // flight. This avoids replacing one unbounded buffer with another inside\n // the downstream Subscription during catchup completion.\n const send = isQueuedStatus(entry)\n ? this.#sendStringifiedDownstream(entry.status).then(() => {})\n : this.#sendChange(entry);\n const promise = send.finally(() => {\n this.#backlogInFlightBytes -= bytes;\n this.#backlogBackpressure.releaseIfUnderLowWater(\n this.#bufferedBacklogBytes,\n );\n });\n inFlight.push({promise, bytes});\n inFlightBytes += bytes;\n\n while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n const next = must(inFlight.shift());\n await next.promise;\n inFlightBytes -= next.bytes;\n }\n }\n\n for (const {promise} of inFlight) {\n await promise;\n }\n } finally {\n this.#backlogDrain = null;\n this.#backlogBackpressure.releaseIfUnderLowWater(\n this.#bufferedBacklogBytes,\n );\n }\n }\n}\n\nclass ByteBackpressureGate {\n readonly highWaterBytes: number;\n readonly #lowWaterBytes: number;\n readonly #waiters: Resolver<void>[] = [];\n\n constructor(highWaterBytes: number, lowWaterRatio: number) {\n this.highWaterBytes = Math.max(1, highWaterBytes);\n this.#lowWaterBytes =\n this.highWaterBytes Math.min(1, Math.max(0, lowWaterRatio));\n }\n\n waitForSpace(bufferedBytes: number): Promise<void> {\n if (bufferedBytes < this.highWaterBytes) {\n return promiseVoid;\n }\n\n // One waiter represents one send() call that has already appended its\n // change. The producer is released when the backlog falls back below the low\n // water mark or the subscriber closes.\n const r = resolver<void>();\n this.#waiters.push(r);\n return r.promise;\n }\n\n releaseIfUnderLowWater(bufferedBytes: number) {\n if (this.#waiters.length === 0 \|\| bufferedBytes > this.#lowWaterBytes) {\n return;\n }\n\n // Use a low water mark so waiting producers are released in batches instead\n // of waking one at a time around the high water boundary.\n this.releaseAll();\n }\n\n releaseAll() {\n const waiters = this.#waiters.splice(0);\n for (const waiter of waiters) {\n waiter.resolve();\n }\n }\n}\n\n/*\n Sent to an aligned subscriber when lag reporting is disabled upstream,\n * signaling that the subscriber is caught up. This is a lag report with a\n * `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured\n * (see `isLagReportingDisabledSignal()` in the replicator's recorder).\n */\nexport const LAG_REPORTING_DISABLED: Status = {\n tag: 'status',\n lagReport: {nextSendTimeMs: 0},\n};\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;AAqDxC,SAAS,eAAe,OAA4C;CAClE,OAAO,CAAC,MAAM,QAAQ,KAAK;AAC7B;AAEA,SAAS,WAAW,OAA6B;CAC/C,OAAO,eAAe,KAAK,IAAI,MAAM,OAAO,SAAS,MAAM,GAAG;AAChE;;;;;;;;AAuBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA;CACA,WAAW;CAEX,YACE,iBACA,IACA,MACA,WACA,YACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAKC,cAAc;EACnB,KAAKC,aAAa,QAAQ,aAAa;EACvC,KAAKQ,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKT,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAKE,SAAS,QAAQ;EACtB,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,qBAAqB,QAAQ;EAClC,KAAKC,wBAAwB,QAAQ;CACvC;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAKH;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAKO;CACd;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKP,YAAY;GACnB,MAAM,UAAU,KAAKA,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAKO,UACzB,KAAKL,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAKE;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAKG,yBAAyB,KAAKX,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAKS,aAAa,QAAQ,KAAKE,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAKV,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAKA,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKM,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKG,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAKf,cACL,iBACA,CAAC,KAAKU,YACN,KAAKM,gBACL,QAAQ,GAAG,KAAK,KAAKR,eACpB,KAAKV,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,GAAG,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,GAAG;IAC7B;GACF;GAEF,IAAI,iBACF,KAAKU,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAKS,gBAAgB,MAAM;GAE7B,OAAO,KAAKC,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,EAAE,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKF,cAAc;GACtB,KAAKA,eAAe;GAEpB,KAAUG,gBAAgB,CAAC,UAAU,EAAC,KAAK,SAAQ,CAAC,CAAC;EACvD;CACF;;;;;;;;;;;CAYA,WAAW,QAAgB;EACzB,IAAI,CAAC,KAAKH,cACR;EAEF,MAAM,aAAyB,CAAC,UAAU,MAAM;EAChD,IAAI,KAAKN,UAAU;GACjB,KAAKG,aAAa,EAAC,QAAQ,WAAW,UAAU,UAAU,EAAC,CAAC;GAC5D;EACF;EACA,KAAUM,gBAAgB,UAAU;CACtC;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKL,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKK,YAAY;EACjB,IAAI,CAAC,KAAKV,UACR,OAAO,KAAKW,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAMN,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKP,aAAa;EAEpB,KAAKS,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAKM,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YAAY;GAI7C,MAAM,QAAQ,IAAI,KAAKd,QAAQ,SAAS;GACxC,IAAI,UAAU,KAAKA,QAAQ;IACzB,KAAKA,SAAS;IACd,KAAKN,SAAS,KAAK;GACrB;EACF;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKoB,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAKC;EACL,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK1B,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAKyB;GACL,KAAKC,iBAAiB;GACtB,KAAKC;EACP;CACF;CAEA,MAAMR,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAKM,YAAY,MAAM,QAAQ;EAC/B,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK1B,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YAAY;IAC7C,MAAM,QAAQ,IAAI,KAAKU,QAAQ,eAAe;IAC9C,IAAI,UAAU,KAAKA,QAAQ;KACzB,KAAKA,SAAS;KACd,KAAKN,SAAS,KAAK;IACrB;GACF;EACF,UAAU;GACR,KAAKqB,YAAY,MAAM,QAAQ;GAC/B,KAAKC,iBAAiB;GACtB,KAAKC,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKF,WAAW,KAAKG;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKf;GACnB,oBAAoB,KAAKiB;GACzB,mBAAmB,KAAKC;EAC1B;EAEF,MAAM,OAAO,KAAKF,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKf;GACnB,oBAAoB,KAAKiB;GACzB,mBAAmB,KAAKC;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAKA,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAKC,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAKD,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAKC,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAKA,oBAAoB;EACzB,OAAO,MAAM,KAAKA;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAKjC,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MAAM,OAAmB,SAAkB;EAGzC,KAAKY,WAAW;EAChB,KAAKsB,gBAAgB;EACrB,KAAK/B,qBAAqB,WAAW;EAGrC,KAAKgC,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAUd,gBAAgB,CAAC,SAAS;GAAC,MAAM;GAAO;EAAO,CAAC,CAAC,EAAE,cAC3D,KAAKpB,YAAY,OAAO,CAC1B;OAEA,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAI4B,gBAAgB;EAClB,OAAO,KAAKjB,UAAU,QAAQ;CAChC;CAEA,IAAIE,wBAAwB;EAI1B,OAAO,KAAKoB,gBAAgB,KAAKE;CACnC;CAEA,IAAIL,sBAAsB;EACxB,OAAO,KAAKjB,wBAAwB,KAAKa;CAC3C;CAEA,aAAa,OAAqB;EAChC,OAAO,KAAKf,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,KAAK;EACxB,KAAKsB,iBAAiB,WAAW,KAAK;EACtC,IAAI,KAAK,aACP,KAAKC,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAK/B,mBAAmB;EAC5C,KAAKA,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAKD,qBAAqB,aAAa,KAAKW,qBAAqB;CAC1E;CAEA,MAAMU,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,QAAQ,KAAKZ,UAAU,MAAM;IACnC,IAAI,CAAC,OAAO;KACV,MAAM,SAAS,KAAKA,aAAa;KACjC,KAAKA,WAAW;KAChB,KAAKsB,gBAAgB;KACrB,KAAK/B,qBAAqB,uBACxB,KAAKW,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAKD,UAAU;MAG7B,KAAKA,WAAW;MAChB,KAAKN,aAAa,IAAI;MACtB,KAAUE,uBAAuB,MAAK,aAAY;OAChD,IAAI,UACF,KAAK,WAAW,sBAAsB;MAE1C,CAAC;KACH;KACA;IACF;IAEA,MAAM,QAAQ,WAAW,KAAK;IAC9B,KAAKyB,iBAAiB;IACtB,KAAKE,yBAAyB;IAC9B,KAAKjC,qBAAqB,uBACxB,KAAKW,qBACP;IAQA,MAAM,WAHO,eAAe,KAAK,IAC7B,KAAKW,2BAA2B,MAAM,MAAM,EAAE,WAAW,CAAC,CAAC,IAC3D,KAAKR,YAAY,KAAK,GACL,cAAc;KACjC,KAAKmB,yBAAyB;KAC9B,KAAKjC,qBAAqB,uBACxB,KAAKW,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKX,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKoB,gBAAgB;GACrB,KAAKpB,qBAAqB,uBACxB,KAAKW,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAKuB,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF;;;;;;;AAQA,IAAa,yBAAiC;CAC5C,KAAK;CACL,WAAW,EAAC,gBAAgB,EAAC;AAC/B"}

package/out/zero-cache/src/services/http-service.d.ts CHANGED Viewed

@@ -5,7 +5,7 @@ import type { Service } from './service.ts';
 export type Options = {
     port: number;
     keepaliveTimeoutMs: number | undefined;
-    readinessGate?: Promise<void> | undefined;
+    readinessGate?: Promise<unknown> | undefined;
 };
 /**
  * Common functionality for all HttpServices. These include:

package/out/zero-cache/src/services/http-service.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IAGvC,aAAa,CAAC,EAAE,OAAO,CAAC,~~IAAI~~,CAAC,GAAG,SAAS,CAAC;~~CAC3C~~,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAMxC,YACE,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAczD;IAGD,SAAS,CAAC,QAAQ,SAAK;IACvB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjC;IAGK,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAoB7B;IAEK,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAM1B;CACF"}
1	+ {"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IAGvC,aAAa,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAMxC,YACE,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAczD;IAGD,SAAS,CAAC,QAAQ,SAAK;IACvB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjC;IAGK,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAoB7B;IAEK,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAM1B;CACF"}

package/out/zero-cache/src/services/http-service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"http-service.js","names":["#fastify","#port","#heartbeatMonitor","#init","#ready"],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n port: number;\n keepaliveTimeoutMs: number \| undefined;\n\n // Wait for the readinessGate to resolve before responding to health checks.\n readinessGate?: Promise<~~void~~> \| undefined;\n};\n\n/*\n Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n readonly id: string;\n protected readonly _lc: LogContext;\n readonly #fastify: FastifyInstance;\n readonly #port: number;\n protected readonly _state: RunningState;\n readonly #heartbeatMonitor: HeartbeatMonitor \| undefined;\n readonly #init: (fastify: FastifyInstance) => void \| Promise<void>;\n\n #ready = false;\n\n constructor(\n id: string,\n lc: LogContext,\n opts: Options,\n init: (fastify: FastifyInstance) => void \| Promise<void>,\n ) {\n const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n this.id = id;\n this._lc = lc.withContext('component', this.id);\n this.#fastify = Fastify();\n this.#port = port;\n this.#init = init;\n this._state = new RunningState(id);\n this.#heartbeatMonitor = keepaliveTimeoutMs\n ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n : undefined;\n\n void readinessGate.then(() => (this.#ready = true));\n }\n\n // Life-cycle hooks for subclass implementations\n protected _onStart() {}\n protected _onStop(): Promise<void> {\n return promiseVoid;\n }\n // start() is used in unit tests.\n // run() is the lifecycle method called by the ServiceRunner.\n async start(): Promise<string> {\n this.#fastify.get('/', (_req, res) => {\n if (this.#ready) {\n res.send('OK');\n }\n });\n this.#fastify.get('/keepalive', ({headers}, res) => {\n this.#heartbeatMonitor?.onHeartbeat(headers);\n if (this.#ready) {\n res.send('OK');\n }\n });\n await this.#init(this.#fastify);\n const address = await this.#fastify.listen({\n host: '::',\n port: this.#port,\n });\n this._lc.info?.(`${this.id} listening at ${address}`);\n this._onStart();\n return address;\n }\n\n async run(): Promise<void> {\n await this.start();\n await this._state.stopped();\n }\n\n async stop(): Promise<void> {\n this._lc.info?.(`${this.id}: no longer accepting connections`);\n this.#heartbeatMonitor?.stop();\n this._state.stop(this._lc);\n await this.#fastify.close();\n await this._onStop();\n }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CAET,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKA,WAAW,QAAQ;EACxB,KAAKC,QAAQ;EACb,KAAKE,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAKD,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAKE,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAKJ,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAKI,QACP,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAKJ,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAKE,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAKE,QACP,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAKD,MAAM,KAAKH,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAKA,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAKC;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAKC,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAKF,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}
1	+ {"version":3,"file":"http-service.js","names":["#fastify","#port","#heartbeatMonitor","#init","#ready"],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n port: number;\n keepaliveTimeoutMs: number \| undefined;\n\n // Wait for the readinessGate to resolve before responding to health checks.\n readinessGate?: Promise<unknown> \| undefined;\n};\n\n/*\n Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n readonly id: string;\n protected readonly _lc: LogContext;\n readonly #fastify: FastifyInstance;\n readonly #port: number;\n protected readonly _state: RunningState;\n readonly #heartbeatMonitor: HeartbeatMonitor \| undefined;\n readonly #init: (fastify: FastifyInstance) => void \| Promise<void>;\n\n #ready = false;\n\n constructor(\n id: string,\n lc: LogContext,\n opts: Options,\n init: (fastify: FastifyInstance) => void \| Promise<void>,\n ) {\n const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n this.id = id;\n this._lc = lc.withContext('component', this.id);\n this.#fastify = Fastify();\n this.#port = port;\n this.#init = init;\n this._state = new RunningState(id);\n this.#heartbeatMonitor = keepaliveTimeoutMs\n ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n : undefined;\n\n void readinessGate.then(() => (this.#ready = true));\n }\n\n // Life-cycle hooks for subclass implementations\n protected _onStart() {}\n protected _onStop(): Promise<void> {\n return promiseVoid;\n }\n // start() is used in unit tests.\n // run() is the lifecycle method called by the ServiceRunner.\n async start(): Promise<string> {\n this.#fastify.get('/', (_req, res) => {\n if (this.#ready) {\n res.send('OK');\n }\n });\n this.#fastify.get('/keepalive', ({headers}, res) => {\n this.#heartbeatMonitor?.onHeartbeat(headers);\n if (this.#ready) {\n res.send('OK');\n }\n });\n await this.#init(this.#fastify);\n const address = await this.#fastify.listen({\n host: '::',\n port: this.#port,\n });\n this._lc.info?.(`${this.id} listening at ${address}`);\n this._onStart();\n return address;\n }\n\n async run(): Promise<void> {\n await this.start();\n await this._state.stopped();\n }\n\n async stop(): Promise<void> {\n this._lc.info?.(`${this.id}: no longer accepting connections`);\n this.#heartbeatMonitor?.stop();\n this._state.stop(this._lc);\n await this.#fastify.close();\n await this._onStop();\n }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CAET,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKA,WAAW,QAAQ;EACxB,KAAKC,QAAQ;EACb,KAAKE,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAKD,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAKE,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAKJ,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAKI,QACP,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAKJ,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAKE,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAKE,QACP,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAKD,MAAM,KAAKH,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAKA,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAKC;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAKC,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAKF,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}

package/out/zero-cache/src/services/litestream/commands.d.ts CHANGED Viewed

@@ -37,6 +37,12 @@ export declare class BackupNotFoundException extends Error {
     static readonly name = "BackupNotFoundException";
     constructor(backupURL: string | undefined);
 }
+/**
+ * The effective backup URL: the `backupURLOverride` if present (i.e. a
+ * lineage-specific location computed by the replication-manager), or
+ * otherwise the configured `backupURL`.
+ */
+export declare function backupDestinationURL(config: Pick<LitestreamConfig, 'backupURL' | 'backupURLOverride'>): string | undefined;
 export declare function tryRestore(lc: LogContext, config: LitestreamConfig, replicaFile: string, replicaConstraints: ReplicaConstraints | undefined, role: LitestreamRole, signal?: AbortSignal, forkBackup?: ForkBackup | undefined): Promise<RestoreAttempt>;
 /**
  * Deletes litestream's local state for `replicaFile` (its default meta

package/out/zero-cache/src/services/litestream/commands.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAK3D,OAAO,EAAC,KAAK,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAOhE,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,yCAAyC,CAAC;AAIzE,OAAO,EAWL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAE1C,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,WAAW,GACX,iBAAiB,GACjB,OAAO,CAAC;AAEZ,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACnC,CAAC;AAKF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;IAEjD,YAAY,SAAS,EAAE,MAAM,GAAG,SAAS,EAExC;CACF;~~AA8FD~~,wBAAsB,UAAU,CAC9B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,IAAI,EAAE,cAAc,EACpB,MAAM,CAAC,EAAE,WAAW,EACpB,UAAU,CAAC,EAAE,UAAU,GAAG,SAAS,GAClC,OAAO,CAAC,cAAc,CAAC,CA2BzB;~~AAgOD~~;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,WAAW,EAAE,MAAM,QAK1D;AA2GD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA8DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}
1	+ {"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAK3D,OAAO,EAAC,KAAK,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAOhE,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,yCAAyC,CAAC;AAIzE,OAAO,EAWL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAE1C,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,WAAW,GACX,iBAAiB,GACjB,OAAO,CAAC;AAEZ,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACnC,CAAC;AAKF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;IAEjD,YAAY,SAAS,EAAE,MAAM,GAAG,SAAS,EAExC;CACF;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,IAAI,CAAC,gBAAgB,EAAE,WAAW,GAAG,mBAAmB,CAAC,GAChE,MAAM,GAAG,SAAS,CAEpB;AA6FD,wBAAsB,UAAU,CAC9B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,IAAI,EAAE,cAAc,EACpB,MAAM,CAAC,EAAE,WAAW,EACpB,UAAU,CAAC,EAAE,UAAU,GAAG,SAAS,GAClC,OAAO,CAAC,cAAc,CAAC,CA2BzB;AAiOD;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,WAAW,EAAE,MAAM,QAK1D;AA2GD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA8DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}

package/out/zero-cache/src/services/litestream/commands.js CHANGED Viewed

@@ -17,8 +17,16 @@ import { basename, dirname, join } from "node:path";
 //#region ../zero-cache/src/services/litestream/commands.ts
 var MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;
 var LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;
+/**
+* The effective backup URL: the `backupURLOverride` if present (i.e. a
+* lineage-specific location computed by the replication-manager), or
+* otherwise the configured `backupURL`.
+*/
+function backupDestinationURL(config) {
+	return config.backupURLOverride ?? config.backupURL;
+}
 function getLitestream(mode, config, replicaFile, logLevelOverride) {
-	const { executable, executableV5, restoreUsingV5, backupUsingV5, backupURL, logLevel, configPath, configPathV5, endpoint, region, port, checkpointThresholdMB, minCheckpointPageCount = checkpointThresholdMB * 256, maxCheckpointPageCount = minCheckpointPageCount * 10, forceCheckpointThresholdMB, incrementalBackupIntervalMinutes, incrementalBackupIntervalSeconds, snapshotBackupIntervalHours, multipartConcurrency, multipartSize } = config;
+	const { executable, executableV5, restoreUsingV5, backupUsingV5, logLevel, configPath, configPathV5, endpoint, region, port, checkpointThresholdMB, minCheckpointPageCount = checkpointThresholdMB * 256, maxCheckpointPageCount = minCheckpointPageCount * 10, forceCheckpointThresholdMB, incrementalBackupIntervalMinutes, incrementalBackupIntervalSeconds, snapshotBackupIntervalHours, multipartConcurrency, multipartSize } = config;
 	const v5 = mode === "restore" && restoreUsingV5 || mode === "replicate" && backupUsingV5;
 	const litestream = (v5 ? executableV5 : executable) ?? must(executable, `Missing --litestream-executable`);
 	const litestreamConfig = v5 ? configPathV5 : configPath;
@@ -28,7 +36,7 @@ function getLitestream(mode, config, replicaFile, logLevelOverride) {
 		env: {
 			...process.env,
 			["ZERO_REPLICA_FILE"]: replicaFile,
-			["ZERO_LITESTREAM_BACKUP_URL"]: must(backupURL),
+			["ZERO_LITESTREAM_BACKUP_URL"]: must(backupDestinationURL(config)),
 			["ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT"]: String(minCheckpointPageCount),
 			["ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT"]: String(maxCheckpointPageCount),
 			["ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES"]: String(incrementalBackupIntervalMinutes),
@@ -59,7 +67,8 @@ async function tryRestore(lc, config, replicaFile, replicaConstraints, role, sig
 	return attempt;
 }
 async function restoreOnce(lc, config, replicaFile, replicaConstraints, role, signal, forkBackup) {
-	const { backupURL, backupUsingV5 } = config;
+	const { backupUsingV5 } = config;
+	const backupURL = backupDestinationURL(config);
 	if (forkBackup && !backupUsingV5) forkBackup = void 0;
 	const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);
 	let result = "error";
@@ -308,7 +317,7 @@ function openValidReplica(lc, replica, constraints) {
 function startReplicaBackupProcess(lc, config, replicaFile) {
 	const { litestream, env } = getLitestream("replicate", config, replicaFile);
 	const attrs = litestreamBackupProcessMetricAttrs(config);
-	lc.info?.(`starting litestream backup to ${config.backupURL}`);
+	lc.info?.(`starting litestream backup to ${backupDestinationURL(config)}`);
 	const start = performance.now();
 	const proc = spawn(litestream, ["replicate"], {
 		env,
@@ -361,7 +370,7 @@ var wsRe = /\s+/;
 async function getLastBackupTime(lc, config, replicaFile) {
 	const [snapshots, wal] = await Promise.all([listBackupCreatedTimes(lc, config, replicaFile, "snapshots"), listBackupCreatedTimes(lc, config, replicaFile, "wal")]);
 	const times = [...snapshots, ...wal];
-	if (times.length === 0) throw new Error(`no snapshots or WAL segments listed at ${config.backupURL}`);
+	if (times.length === 0) throw new Error(`no snapshots or WAL segments listed at ${backupDestinationURL(config)}`);
 	return new Date(Math.max(...times.map((time) => time.getTime())));
 }
 /**
@@ -439,6 +448,6 @@ function parseBackupCreatedTimes(lc, command, output) {
 	return times;
 }
 //#endregion
-export { deleteLitestreamMetaDir, getLastBackupTime, startReplicaBackupProcess, tryRestore };
+export { backupDestinationURL, deleteLitestreamMetaDir, getLastBackupTime, startReplicaBackupProcess, tryRestore };
 //# sourceMappingURL=commands.js.map

package/out/zero-cache/src/services/litestream/commands.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {InitCleanup} from '../change-source/common/init-cleanup.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreValidationDuration,\n type LitestreamRole,\n} from './metrics.ts';\nimport {WalKeeper} from './wal-keeper.ts';\n\nexport type ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\nexport type RestoreResult =\n \| 'success'\n \| 'no_backup'\n \| 'invalid_replica'\n \| 'error';\n\nexport type RestoreAttempt = {\n restored: boolean;\n backupURL: string \| undefined;\n result: RestoreResult;\n /*\n Set when the restore prepared the backup to continue from the restored\n * replica (see {@link ForkBackup}). It is registered with the\n * `cleanup`; otherwise the caller must release it (see {@link WalKeeper}).\n /\n walKeeper?: WalKeeper \| undefined;\n};\n\n/\n Prepares the restored replica to continue from a forked backup\n * instead of starting a new lineage with a full snapshot.\n /\nexport type ForkBackup = {\n /* The restore also forks its backup into the (new, empty) one at this URL. /\n url: string;\n /\n Registers the resulting {@link WalKeeper} to be released if the\n * initialization fails.\n /\n cleanup?: InitCleanup \| undefined;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string \| undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\nfunction getLitestream(\n mode: 'restore' \| 'replicate',\n config: LitestreamConfig,\n replicaFile: string,\n logLevelOverride?: LogLevel,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupUsingV5,\n backupURL,\n logLevel,\n configPath,\n configPathV5,\n endpoint,\n region,\n port,\n checkpointThresholdMB,\n // Note: This assumes the default page size. If page size configuration\n // is added, this computation must be adjusted accordingly.\n minCheckpointPageCount = checkpointThresholdMB 256, // 1 MiB / 4 KiB page size\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n forceCheckpointThresholdMB,\n incrementalBackupIntervalMinutes,\n incrementalBackupIntervalSeconds,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config;\n\n const v5 =\n (mode === 'restore' && restoreUsingV5) \|\|\n (mode === 'replicate' && backupUsingV5);\n const litestream =\n (v5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n const litestreamConfig = v5 ? configPathV5 : configPath;\n // Disable truncate-page-n if forced checkpoints are enabled,\n // and otherwise use litestream's default.\n const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: replicaFile,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes, // v3 only\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n incrementalBackupIntervalSeconds, // v5 only\n ),\n // v5 only\n ['ZERO_LITESTREAM_MONITOR_INTERVAL_SECONDS']: String(\n // zero-cache explicitly requests syncs at the incremental backup\n // interval (see LitestreamSyncRequester), and litestream's\n // own monitor becomes a backstop that keeps backups and checkpoints\n // progressing if zero-cache stops requesting them.\n 2 * incrementalBackupIntervalSeconds,\n ),\n ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours,\n ),\n ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: 'json',\n ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n ['LITESTREAM_CONFIG']: litestreamConfig,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n ['LITESTREAM_COMPACTION_SERIALIZE']: 'false',\n },\n };\n}\n\nexport async function tryRestore(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints \| undefined,\n role: LitestreamRole,\n signal?: AbortSignal,\n forkBackup?: ForkBackup \| undefined,\n): Promise<RestoreAttempt> {\n const {reusedExisting, ...attempt} = await restoreOnce(\n lc,\n config,\n replicaFile,\n replicaConstraints,\n role,\n signal,\n forkBackup,\n );\n if (attempt.result === 'invalid_replica' && reusedExisting) {\n // `-if-db-not-exists` made the restore a no-op, and the existing local\n // replica (e.g. one left behind by a failed initial sync of a different\n // generation) was invalid. It has since been deleted, so restore from the\n // backup.\n const {reusedExisting: _, ...retry} = await restoreOnce(\n lc,\n config,\n replicaFile,\n replicaConstraints,\n role,\n signal,\n forkBackup,\n );\n return retry;\n }\n return attempt;\n}\n\nasync function restoreOnce(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints \| undefined,\n role: LitestreamRole,\n signal: AbortSignal \| undefined,\n forkBackup: ForkBackup \| undefined,\n): Promise<RestoreAttempt & {reusedExisting: boolean}> {\n const {backupURL, backupUsingV5} = config;\n if (forkBackup && !backupUsingV5) {\n forkBackup = undefined; // Ignore forkBackup if v5 is disabled.\n }\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n let result: RestoreResult = 'error';\n try {\n logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n deleteRestoreTempFiles(replicaFile);\n const reusedExisting = existsSync(replicaFile);\n if (!reusedExisting) {\n // The local litestream state (and anything recorded in it) describes a\n // replica that no longer exists. Discard it before the restore, which\n // (if it seeds a backup) writes the state that belongs to the new one.\n deleteLitestreamMetaDir(replicaFile);\n }\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n replicaFile,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided how to handle the failure. The captured output is\n // included in the thrown error so the caller can log the final outcome.\n // See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n ...(forkBackup ? ['-json', '-fork-to-url', forkBackup.url] : []),\n replicaFile,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, signal},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n // Output AND capture the stdout and stderr, the former for observability\n // and the latter for error handling.\n proc.stdout.on('data', chunk => {\n process.stdout.write(chunk);\n stdout += chunk;\n });\n proc.stderr.on('data', chunk => {\n process.stderr.write(chunk);\n stderr += chunk;\n });\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n const output = [stdout, stderr]\n .map(value => value.trim())\n .filter(Boolean)\n .join('\\n');\n throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n cause: e,\n });\n }\n if (!existsSync(replicaFile)) {\n result = 'no_backup';\n return {restored: false, backupURL, result, reusedExisting};\n }\n const validationStart = performance.now();\n // On success, the validating connection is kept open until it is either\n // closed or kept as the WalKeeper. Until then, its close would be the last\n // one, which checkpoints and deletes the WAL that a seeded backup continues\n // from.\n let db = openValidReplica(lc, replicaFile, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: db ? 'success' : 'invalid_replica'},\n );\n if (!db) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(replicaFile);\n deleteChangeLogDB(replicaFile);\n // The litestream state (e.g. the anchor of a seeded backup) describes\n // the deleted replica.\n deleteLitestreamMetaDir(replicaFile);\n return {restored: false, backupURL, result, reusedExisting};\n }\n try {\n result = 'success';\n if (!reusedExisting) {\n // This restore materialized the replica file, so any change log beside it\n // was written against a replica that is no longer there. Reconciliation\n // is the safety net — a restored replica keeps its replicaVersion, so the\n // rule falls through to truncate-or-reseed on head mismatch — but the log\n // is a cache and deleting it here is the explicit statement that nothing\n // in it survives the restore.\n //\n // A restore that reuses an existing replica (`-if-db-not-exists` made it\n // a no-op) deliberately keeps the log: that is the process-restart path,\n // where the log is still the one written beside this exact replica and\n // discarding it would cost every reconnecting subscriber a `too-old`.\n deleteChangeLogDB(replicaFile);\n litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n ...attrs,\n result: 'success',\n });\n }\n // Last, so that nothing can fail after the WalKeeper is created.\n const walKeeper =\n forkBackup && !reusedExisting && restoreForked(lc, stdout)\n ? new WalKeeper(lc, db)\n : undefined;\n if (walKeeper) {\n db = undefined; // now owned by the WalKeeper\n forkBackup?.cleanup?.onFailure('WAL keeper', () =>\n walKeeper.release('initialization failed'),\n );\n }\n return {restored: true, backupURL, result, reusedExisting, walKeeper};\n } finally {\n db?.close();\n }\n } finally {\n try {\n deleteRestoreTempFiles(replicaFile);\n } catch (e) {\n lc.warn?.('Unable to clean up Litestream restore temporary files', {\n replicaFile,\n error: String(e),\n });\n }\n logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n litestreamRestoreAttempts().add(1, {...attrs, result});\n }\n}\n\n// From litestream/cmd/litestream/restore.go\nconst restoreForkOutputSchema = v.object({\n fork: v\n .object({\n forked: v.boolean(),\n txid: v.string().optional(),\n })\n .optional(),\n});\n\ntype RestoreForkOutput = v.Infer<typeof restoreForkOutputSchema>;\n\n/*\n Whether the `-json` output of `litestream restore -fork-to-url` reports that\n * the backup was forked. A restore plan that cannot fork the backup is\n * restored without forking.\n /\nfunction restoreForked(lc: LogContext, stdout: string): boolean {\n const start = stdout.indexOf('{');\n const end = stdout.lastIndexOf('}');\n if (start < 0 \|\| end < start) {\n lc.warn?.('litestream restore did not report a fork result');\n return false;\n }\n let output: RestoreForkOutput;\n try {\n output = v.parse(\n JSON.parse(stdout.slice(start, end + 1)),\n restoreForkOutputSchema,\n 'passthrough',\n );\n } catch (e) {\n lc.warn?.('unable to parse the litestream restore result', e);\n return false;\n }\n const forked = output.fork?.forked === true;\n lc.info?.(\n forked\n ? `forked the backup at txid ${output.fork?.txid}`\n : 'restored without forking the backup',\n );\n return forked;\n}\n\n/\n Deletes litestream's local state for `replicaFile` (its default meta\n * directory, `.<file>-litestream`).\n /\nexport function deleteLitestreamMetaDir(replicaFile: string) {\n rmSync(join(dirname(replicaFile), `.${basename(replicaFile)}-litestream`), {\n recursive: true,\n force: true,\n });\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n const temporaryReplicaFile = `${replicaFile}.tmp`;\n deleteLiteDB(temporaryReplicaFile);\n\n // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n const directory = dirname(temporaryReplicaFile);\n if (!existsSync(directory)) {\n return;\n }\n const prefix = `${basename(temporaryReplicaFile)}-`;\n for (const name of readdirSync(directory)) {\n if (!name.startsWith(prefix) \|\| !name.endsWith('-wal')) {\n continue;\n }\n const index = name.slice(prefix.length, -'-wal'.length);\n if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n rmSync(join(directory, name), {force: true});\n }\n }\n}\n\nfunction logRestoreDirectoryContents(\n lc: LogContext,\n replicaFile: string,\n phase: 'before-temp-cleanup' \| 'after-temp-cleanup',\n) {\n const directory = dirname(replicaFile);\n try {\n const names = readdirSync(directory).sort();\n const entries = names\n .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n lc.info?.('Litestream restore directory contents', {\n phase,\n directory,\n replicaFile,\n entryCount: names.length,\n omittedEntryCount: Math.max(\n 0,\n names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n ),\n entries,\n });\n } catch (e) {\n lc.warn?.('Unable to inspect Litestream restore directory', {\n phase,\n directory,\n replicaFile,\n error: String(e),\n });\n }\n}\n\n/\n Opens and validates the restored replica. Returns the open connection if\n * the replica is valid, which the caller must close; otherwise closes it and\n * returns `undefined`.\n /\nfunction openValidReplica(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints \| undefined,\n): Database \| undefined {\n let db: Database \| undefined;\n try {\n // Note: Open the database and read the subscription state as a\n // sanity / corruption check, even if there are no constraints.\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (constraints) {\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return undefined;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return undefined;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n }\n const valid = db;\n db = undefined; // owned by the caller\n return valid;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return undefined;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${config.backupURL}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' \| 'error' \| 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/\n Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n \n This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * believes an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n \n Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n /\nexport async function getLastBackupTime(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${config.backupURL}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/\n Runs `litestream <snapshots\|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n \n ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n /\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n command: 'snapshots' \| 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' \| 'empty' \| 'timeout' \| 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const proc = spawn(litestream, [command, replicaFile], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/\n Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n \n Exported for testing.\n /\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' \| 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 \|\|\n created === undefined \|\|\n created === 'created' / header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwEA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;AAUjC,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,WACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAG7C,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,SAAS;IAC7C,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IAEC,6CAA6C,OAK5C,IAAI,gCACN;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;IACpD,oCAAoC;EACvC;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACA,QACA,YACyB;CACzB,MAAM,EAAC,gBAAgB,GAAG,YAAW,MAAM,YACzC,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;CACA,IAAI,QAAQ,WAAW,qBAAqB,gBAAgB;EAK1D,MAAM,EAAC,gBAAgB,GAAG,GAAG,UAAS,MAAM,YAC1C,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;EACA,OAAO;CACT;CACA,OAAO;AACT;AAEA,eAAe,YACb,IACA,QACA,aACA,oBACA,MACA,QACA,YACqD;CACrD,MAAM,EAAC,WAAW,kBAAiB;CACnC,IAAI,cAAc,CAAC,eACjB,aAAa,KAAA;CAEf,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,iBAAiB,WAAW,WAAW;EAC7C,IAAI,CAAC,gBAIH,wBAAwB,WAAW;EAErC,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,GAAI,aAAa;IAAC;IAAS;IAAgB,WAAW;GAAG,IAAI,CAAC;GAC9D;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;GAAM;EAAM,CACpE;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,MAAM,kBAAkB,YAAY,IAAI;EAKxC,IAAI,KAAK,iBAAiB,IAAI,aAAa,kBAAkB;EAC7D,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,KAAK,YAAY;EAAiB,CACvD;EACA,IAAI,CAAC,IAAI;GACP,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAG7B,wBAAwB,WAAW;GACnC,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,IAAI;GACF,SAAS;GACT,IAAI,CAAC,gBAAgB;IAYnB,kBAAkB,WAAW;IAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;KAC1D,GAAG;KACH,QAAQ;IACV,CAAC;GACH;GAEA,MAAM,YACJ,cAAc,CAAC,kBAAkB,cAAc,IAAI,MAAM,IACrD,IAAI,UAAU,IAAI,EAAE,IACpB,KAAA;GACN,IAAI,WAAW;IACb,KAAK,KAAA;IACL,YAAY,SAAS,UAAU,oBAC7B,UAAU,QAAQ,uBAAuB,CAC3C;GACF;GACA,OAAO;IAAC,UAAU;IAAM;IAAW;IAAQ;IAAgB;GAAS;EACtE,UAAU;GACR,IAAI,MAAM;EACZ;CACF,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAGA,IAAM,0BAA0B,eAAE,OAAO,EACvC,MAAM,eACH,OAAO;CACN,QAAQ,eAAE,QAAQ;CAClB,MAAM,eAAE,OAAO,EAAE,SAAS;AAC5B,CAAC,EACA,SAAS,EACd,CAAC;;;;;;AASD,SAAS,cAAc,IAAgB,QAAyB;CAC9D,MAAM,QAAQ,OAAO,QAAQ,GAAG;CAChC,MAAM,MAAM,OAAO,YAAY,GAAG;CAClC,IAAI,QAAQ,KAAK,MAAM,OAAO;EAC5B,GAAG,OAAO,iDAAiD;EAC3D,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,MACP,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,CAAC,CAAC,GACvC,yBACA,aACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,iDAAiD,CAAC;EAC5D,OAAO;CACT;CACA,MAAM,SAAS,OAAO,MAAM,WAAW;CACvC,GAAG,OACD,SACI,6BAA6B,OAAO,MAAM,SAC1C,qCACN;CACA,OAAO;AACT;;;;;AAMA,SAAgB,wBAAwB,aAAqB;CAC3D,OAAO,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,EAAE,YAAY,GAAG;EACzE,WAAW;EACX,OAAO;CACT,CAAC;AACH;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;;;;;;AAOA,SAAS,iBACP,IACA,SACA,aACsB;CACtB,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA;GACF;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA;GACF;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,MAAM,QAAQ;EACd,KAAK,KAAA;EACL,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW;CAC7D,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WACnD;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
1	+ {"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {InitCleanup} from '../change-source/common/init-cleanup.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreValidationDuration,\n type LitestreamRole,\n} from './metrics.ts';\nimport {WalKeeper} from './wal-keeper.ts';\n\nexport type ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\nexport type RestoreResult =\n \| 'success'\n \| 'no_backup'\n \| 'invalid_replica'\n \| 'error';\n\nexport type RestoreAttempt = {\n restored: boolean;\n backupURL: string \| undefined;\n result: RestoreResult;\n /*\n Set when the restore prepared the backup to continue from the restored\n * replica (see {@link ForkBackup}). It is registered with the\n * `cleanup`; otherwise the caller must release it (see {@link WalKeeper}).\n /\n walKeeper?: WalKeeper \| undefined;\n};\n\n/\n Prepares the restored replica to continue from a forked backup\n * instead of starting a new lineage with a full snapshot.\n /\nexport type ForkBackup = {\n /* The restore also forks its backup into the (new, empty) one at this URL. /\n url: string;\n /\n Registers the resulting {@link WalKeeper} to be released if the\n * initialization fails.\n /\n cleanup?: InitCleanup \| undefined;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string \| undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\n/\n The effective backup URL: the `backupURLOverride` if present (i.e. a\n * lineage-specific location computed by the replication-manager), or\n * otherwise the configured `backupURL`.\n /\nexport function backupDestinationURL(\n config: Pick<LitestreamConfig, 'backupURL' \| 'backupURLOverride'>,\n): string \| undefined {\n return config.backupURLOverride ?? config.backupURL;\n}\n\nfunction getLitestream(\n mode: 'restore' \| 'replicate',\n config: LitestreamConfig,\n replicaFile: string,\n logLevelOverride?: LogLevel,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupUsingV5,\n logLevel,\n configPath,\n configPathV5,\n endpoint,\n region,\n port,\n checkpointThresholdMB,\n // Note: This assumes the default page size. If page size configuration\n // is added, this computation must be adjusted accordingly.\n minCheckpointPageCount = checkpointThresholdMB 256, // 1 MiB / 4 KiB page size\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n forceCheckpointThresholdMB,\n incrementalBackupIntervalMinutes,\n incrementalBackupIntervalSeconds,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config;\n\n const v5 =\n (mode === 'restore' && restoreUsingV5) \|\|\n (mode === 'replicate' && backupUsingV5);\n const litestream =\n (v5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n const litestreamConfig = v5 ? configPathV5 : configPath;\n // Disable truncate-page-n if forced checkpoints are enabled,\n // and otherwise use litestream's default.\n const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: replicaFile,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupDestinationURL(config)),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes, // v3 only\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n incrementalBackupIntervalSeconds, // v5 only\n ),\n // v5 only\n ['ZERO_LITESTREAM_MONITOR_INTERVAL_SECONDS']: String(\n // zero-cache explicitly requests syncs at the incremental backup\n // interval (see LitestreamSyncRequester), and litestream's\n // own monitor becomes a backstop that keeps backups and checkpoints\n // progressing if zero-cache stops requesting them.\n 2 * incrementalBackupIntervalSeconds,\n ),\n ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours,\n ),\n ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: 'json',\n ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n ['LITESTREAM_CONFIG']: litestreamConfig,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n ['LITESTREAM_COMPACTION_SERIALIZE']: 'false',\n },\n };\n}\n\nexport async function tryRestore(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints \| undefined,\n role: LitestreamRole,\n signal?: AbortSignal,\n forkBackup?: ForkBackup \| undefined,\n): Promise<RestoreAttempt> {\n const {reusedExisting, ...attempt} = await restoreOnce(\n lc,\n config,\n replicaFile,\n replicaConstraints,\n role,\n signal,\n forkBackup,\n );\n if (attempt.result === 'invalid_replica' && reusedExisting) {\n // `-if-db-not-exists` made the restore a no-op, and the existing local\n // replica (e.g. one left behind by a failed initial sync of a different\n // generation) was invalid. It has since been deleted, so restore from the\n // backup.\n const {reusedExisting: _, ...retry} = await restoreOnce(\n lc,\n config,\n replicaFile,\n replicaConstraints,\n role,\n signal,\n forkBackup,\n );\n return retry;\n }\n return attempt;\n}\n\nasync function restoreOnce(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints \| undefined,\n role: LitestreamRole,\n signal: AbortSignal \| undefined,\n forkBackup: ForkBackup \| undefined,\n): Promise<RestoreAttempt & {reusedExisting: boolean}> {\n const {backupUsingV5} = config;\n const backupURL = backupDestinationURL(config);\n if (forkBackup && !backupUsingV5) {\n forkBackup = undefined; // Ignore forkBackup if v5 is disabled.\n }\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n let result: RestoreResult = 'error';\n try {\n logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n deleteRestoreTempFiles(replicaFile);\n const reusedExisting = existsSync(replicaFile);\n if (!reusedExisting) {\n // The local litestream state (and anything recorded in it) describes a\n // replica that no longer exists. Discard it before the restore, which\n // (if it seeds a backup) writes the state that belongs to the new one.\n deleteLitestreamMetaDir(replicaFile);\n }\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n replicaFile,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided how to handle the failure. The captured output is\n // included in the thrown error so the caller can log the final outcome.\n // See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n ...(forkBackup ? ['-json', '-fork-to-url', forkBackup.url] : []),\n replicaFile,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, signal},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n // Output AND capture the stdout and stderr, the former for observability\n // and the latter for error handling.\n proc.stdout.on('data', chunk => {\n process.stdout.write(chunk);\n stdout += chunk;\n });\n proc.stderr.on('data', chunk => {\n process.stderr.write(chunk);\n stderr += chunk;\n });\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n const output = [stdout, stderr]\n .map(value => value.trim())\n .filter(Boolean)\n .join('\\n');\n throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n cause: e,\n });\n }\n if (!existsSync(replicaFile)) {\n result = 'no_backup';\n return {restored: false, backupURL, result, reusedExisting};\n }\n const validationStart = performance.now();\n // On success, the validating connection is kept open until it is either\n // closed or kept as the WalKeeper. Until then, its close would be the last\n // one, which checkpoints and deletes the WAL that a seeded backup continues\n // from.\n let db = openValidReplica(lc, replicaFile, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: db ? 'success' : 'invalid_replica'},\n );\n if (!db) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(replicaFile);\n deleteChangeLogDB(replicaFile);\n // The litestream state (e.g. the anchor of a seeded backup) describes\n // the deleted replica.\n deleteLitestreamMetaDir(replicaFile);\n return {restored: false, backupURL, result, reusedExisting};\n }\n try {\n result = 'success';\n if (!reusedExisting) {\n // This restore materialized the replica file, so any change log beside it\n // was written against a replica that is no longer there. Reconciliation\n // is the safety net — a restored replica keeps its replicaVersion, so the\n // rule falls through to truncate-or-reseed on head mismatch — but the log\n // is a cache and deleting it here is the explicit statement that nothing\n // in it survives the restore.\n //\n // A restore that reuses an existing replica (`-if-db-not-exists` made it\n // a no-op) deliberately keeps the log: that is the process-restart path,\n // where the log is still the one written beside this exact replica and\n // discarding it would cost every reconnecting subscriber a `too-old`.\n deleteChangeLogDB(replicaFile);\n litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n ...attrs,\n result: 'success',\n });\n }\n // Last, so that nothing can fail after the WalKeeper is created.\n const walKeeper =\n forkBackup && !reusedExisting && restoreForked(lc, stdout)\n ? new WalKeeper(lc, db)\n : undefined;\n if (walKeeper) {\n db = undefined; // now owned by the WalKeeper\n forkBackup?.cleanup?.onFailure('WAL keeper', () =>\n walKeeper.release('initialization failed'),\n );\n }\n return {restored: true, backupURL, result, reusedExisting, walKeeper};\n } finally {\n db?.close();\n }\n } finally {\n try {\n deleteRestoreTempFiles(replicaFile);\n } catch (e) {\n lc.warn?.('Unable to clean up Litestream restore temporary files', {\n replicaFile,\n error: String(e),\n });\n }\n logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n litestreamRestoreAttempts().add(1, {...attrs, result});\n }\n}\n\n// From litestream/cmd/litestream/restore.go\nconst restoreForkOutputSchema = v.object({\n fork: v\n .object({\n forked: v.boolean(),\n txid: v.string().optional(),\n })\n .optional(),\n});\n\ntype RestoreForkOutput = v.Infer<typeof restoreForkOutputSchema>;\n\n/*\n Whether the `-json` output of `litestream restore -fork-to-url` reports that\n * the backup was forked. A restore plan that cannot fork the backup is\n * restored without forking.\n /\nfunction restoreForked(lc: LogContext, stdout: string): boolean {\n const start = stdout.indexOf('{');\n const end = stdout.lastIndexOf('}');\n if (start < 0 \|\| end < start) {\n lc.warn?.('litestream restore did not report a fork result');\n return false;\n }\n let output: RestoreForkOutput;\n try {\n output = v.parse(\n JSON.parse(stdout.slice(start, end + 1)),\n restoreForkOutputSchema,\n 'passthrough',\n );\n } catch (e) {\n lc.warn?.('unable to parse the litestream restore result', e);\n return false;\n }\n const forked = output.fork?.forked === true;\n lc.info?.(\n forked\n ? `forked the backup at txid ${output.fork?.txid}`\n : 'restored without forking the backup',\n );\n return forked;\n}\n\n/\n Deletes litestream's local state for `replicaFile` (its default meta\n * directory, `.<file>-litestream`).\n /\nexport function deleteLitestreamMetaDir(replicaFile: string) {\n rmSync(join(dirname(replicaFile), `.${basename(replicaFile)}-litestream`), {\n recursive: true,\n force: true,\n });\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n const temporaryReplicaFile = `${replicaFile}.tmp`;\n deleteLiteDB(temporaryReplicaFile);\n\n // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n const directory = dirname(temporaryReplicaFile);\n if (!existsSync(directory)) {\n return;\n }\n const prefix = `${basename(temporaryReplicaFile)}-`;\n for (const name of readdirSync(directory)) {\n if (!name.startsWith(prefix) \|\| !name.endsWith('-wal')) {\n continue;\n }\n const index = name.slice(prefix.length, -'-wal'.length);\n if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n rmSync(join(directory, name), {force: true});\n }\n }\n}\n\nfunction logRestoreDirectoryContents(\n lc: LogContext,\n replicaFile: string,\n phase: 'before-temp-cleanup' \| 'after-temp-cleanup',\n) {\n const directory = dirname(replicaFile);\n try {\n const names = readdirSync(directory).sort();\n const entries = names\n .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n lc.info?.('Litestream restore directory contents', {\n phase,\n directory,\n replicaFile,\n entryCount: names.length,\n omittedEntryCount: Math.max(\n 0,\n names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n ),\n entries,\n });\n } catch (e) {\n lc.warn?.('Unable to inspect Litestream restore directory', {\n phase,\n directory,\n replicaFile,\n error: String(e),\n });\n }\n}\n\n/\n Opens and validates the restored replica. Returns the open connection if\n * the replica is valid, which the caller must close; otherwise closes it and\n * returns `undefined`.\n /\nfunction openValidReplica(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints \| undefined,\n): Database \| undefined {\n let db: Database \| undefined;\n try {\n // Note: Open the database and read the subscription state as a\n // sanity / corruption check, even if there are no constraints.\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (constraints) {\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return undefined;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return undefined;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n }\n const valid = db;\n db = undefined; // owned by the caller\n return valid;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return undefined;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${backupDestinationURL(config)}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' \| 'error' \| 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/\n Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n \n This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * believes an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n \n Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n /\nexport async function getLastBackupTime(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${backupDestinationURL(config)}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/\n Runs `litestream <snapshots\|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n \n ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n /\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n command: 'snapshots' \| 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' \| 'empty' \| 'timeout' \| 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const proc = spawn(litestream, [command, replicaFile], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/\n Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n \n Exported for testing.\n /\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' \| 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 \|\|\n created === undefined \|\|\n created === 'created' / header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwEA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;;;;;;AAejC,SAAgB,qBACd,QACoB;CACpB,OAAO,OAAO,qBAAqB,OAAO;AAC5C;AAEA,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAG7C,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,qBAAqB,MAAM,CAAC;IAChE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IAEC,6CAA6C,OAK5C,IAAI,gCACN;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;IACpD,oCAAoC;EACvC;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACA,QACA,YACyB;CACzB,MAAM,EAAC,gBAAgB,GAAG,YAAW,MAAM,YACzC,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;CACA,IAAI,QAAQ,WAAW,qBAAqB,gBAAgB;EAK1D,MAAM,EAAC,gBAAgB,GAAG,GAAG,UAAS,MAAM,YAC1C,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;EACA,OAAO;CACT;CACA,OAAO;AACT;AAEA,eAAe,YACb,IACA,QACA,aACA,oBACA,MACA,QACA,YACqD;CACrD,MAAM,EAAC,kBAAiB;CACxB,MAAM,YAAY,qBAAqB,MAAM;CAC7C,IAAI,cAAc,CAAC,eACjB,aAAa,KAAA;CAEf,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,iBAAiB,WAAW,WAAW;EAC7C,IAAI,CAAC,gBAIH,wBAAwB,WAAW;EAErC,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,GAAI,aAAa;IAAC;IAAS;IAAgB,WAAW;GAAG,IAAI,CAAC;GAC9D;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;GAAM;EAAM,CACpE;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,MAAM,kBAAkB,YAAY,IAAI;EAKxC,IAAI,KAAK,iBAAiB,IAAI,aAAa,kBAAkB;EAC7D,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,KAAK,YAAY;EAAiB,CACvD;EACA,IAAI,CAAC,IAAI;GACP,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAG7B,wBAAwB,WAAW;GACnC,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,IAAI;GACF,SAAS;GACT,IAAI,CAAC,gBAAgB;IAYnB,kBAAkB,WAAW;IAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;KAC1D,GAAG;KACH,QAAQ;IACV,CAAC;GACH;GAEA,MAAM,YACJ,cAAc,CAAC,kBAAkB,cAAc,IAAI,MAAM,IACrD,IAAI,UAAU,IAAI,EAAE,IACpB,KAAA;GACN,IAAI,WAAW;IACb,KAAK,KAAA;IACL,YAAY,SAAS,UAAU,oBAC7B,UAAU,QAAQ,uBAAuB,CAC3C;GACF;GACA,OAAO;IAAC,UAAU;IAAM;IAAW;IAAQ;IAAgB;GAAS;EACtE,UAAU;GACR,IAAI,MAAM;EACZ;CACF,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAGA,IAAM,0BAA0B,eAAE,OAAO,EACvC,MAAM,eACH,OAAO;CACN,QAAQ,eAAE,QAAQ;CAClB,MAAM,eAAE,OAAO,EAAE,SAAS;AAC5B,CAAC,EACA,SAAS,EACd,CAAC;;;;;;AASD,SAAS,cAAc,IAAgB,QAAyB;CAC9D,MAAM,QAAQ,OAAO,QAAQ,GAAG;CAChC,MAAM,MAAM,OAAO,YAAY,GAAG;CAClC,IAAI,QAAQ,KAAK,MAAM,OAAO;EAC5B,GAAG,OAAO,iDAAiD;EAC3D,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,MACP,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,CAAC,CAAC,GACvC,yBACA,aACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,iDAAiD,CAAC;EAC5D,OAAO;CACT;CACA,MAAM,SAAS,OAAO,MAAM,WAAW;CACvC,GAAG,OACD,SACI,6BAA6B,OAAO,MAAM,SAC1C,qCACN;CACA,OAAO;AACT;;;;;AAMA,SAAgB,wBAAwB,aAAqB;CAC3D,OAAO,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,EAAE,YAAY,GAAG;EACzE,WAAW;EACX,OAAO;CACT,CAAC;AACH;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;;;;;;AAOA,SAAS,iBACP,IACA,SACA,aACsB;CACtB,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA;GACF;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA;GACF;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,MAAM,QAAQ;EACd,KAAK,KAAA;EACL,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,qBAAqB,MAAM,GAAG;CACzE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,qBAAqB,MAAM,GACvE;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts CHANGED Viewed

@@ -4,6 +4,7 @@ import type { ReservationFollowup } from '../change-streamer/change-streamer-htt
 import { type ChangeStreamer } from '../change-streamer/change-streamer.ts';
 import { type ReplicationStatusPublisher } from './replication-status.ts';
 import type { ReplicaState, ReplicatorMode } from './replicator.ts';
+import { type ReplicationLagReadinessOptions } from './reporter/recorder.ts';
 import type { WriteWorkerClient } from './write-worker-client.ts';
 /**
  * The {@link IncrementalSyncer} manages a logical replication stream from upstream,
@@ -14,7 +15,8 @@ import type { WriteWorkerClient } from './write-worker-client.ts';
  */
 export declare class IncrementalSyncer {
     #private;
-    constructor(lc: LogContext, taskID: string, id: string, changeStreamer: ChangeStreamer, worker: WriteWorkerClient, mode: ReplicatorMode, statusPublisher: ReplicationStatusPublisher | null, initialConnection?: ReservationFollowup);
+    constructor(lc: LogContext, taskID: string, id: string, changeStreamer: ChangeStreamer, worker: WriteWorkerClient, mode: ReplicatorMode, statusPublisher: ReplicationStatusPublisher | null, replicationLagReadinessOptions?: ReplicationLagReadinessOptions, initialConnection?: ReservationFollowup);
+    ready(): Promise<void>;
     run(): Promise<void>;
     subscribe(): Source<ReplicaState>;
     stop(lc: LogContext, err?: unknown): void;