@powerhousedao/reactor 6.2.3-dev.32 → 6.2.3-dev.33

package/dist/{document-indexer-BVTGbEB0.js → document-indexer-I6i_YzsR.js}

@@ -1,7 +1,278 @@
-import { ot as ReactorEventTypes, ut as yieldToMain } from "./forwarding-logger-CnuVDVze.js";
+import { C as SettledWatermark, at as DocumentNotFoundError, mt as ReactorEventTypes, vt as yieldToMain, w as createKyselyWatermarkProbe } from "./forwarding-logger-BkwlvD9W.js";
 import { createAuthState, isDenied } from "@powerhousedao/shared/document-model";
 import { v4 } from "uuid";
 import { childLogger } from "document-model";
+//#region src/catch-up/scheduler.ts
+const noHooks = {
+	onSwept: () => {},
+	describeSessions: () => Promise.resolve([])
+};
+function isCatchUpConsumer(value) {
+	if (typeof value !== "object" || value === null) return false;
+	const candidate = value;
+	return typeof candidate.sweep === "function" && typeof candidate.consumerId === "string" && typeof candidate.appliedThrough === "number";
+}
+/** Periodically sweeps every consumer up to the settled watermark. */
+var CatchUpScheduler = class {
+	watermark;
+	operationIndex;
+	config;
+	logger;
+	hooks;
+	fixed = [];
+	sources = [];
+	sourced = /* @__PURE__ */ new Map();
+	timer;
+	running;
+	started = false;
+	stopped = false;
+	warnedStallSince;
+	statusSources = [];
+	constructor(watermark, operationIndex, config, logger, hooks = noHooks) {
+		this.watermark = watermark;
+		this.operationIndex = operationIndex;
+		this.config = config;
+		this.logger = logger;
+		this.hooks = hooks;
+	}
+	get settledWatermark() {
+		return this.watermark;
+	}
+	addConsumer(consumer, thread) {
+		if (this.fixed.some((entry) => entry.consumer === consumer)) return;
+		this.fixed.push({
+			consumer,
+			thread,
+			lastAdvanceUtcMs: Date.now(),
+			warnedHeld: false
+		});
+		this.ensureTimer();
+	}
+	/** Consumers read on every tick, so models registered later are swept too. */
+	addSource(consumers, thread) {
+		this.sources.push({
+			consumers,
+			thread
+		});
+		this.ensureTimer();
+	}
+	/** Consumers another thread sweeps, reported in status(). */
+	addStatusSource(source) {
+		this.statusSources.push(source);
+	}
+	start() {
+		this.started = true;
+		this.ensureTimer();
+	}
+	async stop() {
+		this.stopped = true;
+		if (this.timer !== void 0) {
+			clearInterval(this.timer);
+			this.timer = void 0;
+		}
+		if (this.running !== void 0) await this.running.catch(() => []);
+	}
+	async sweepNow() {
+		if (this.running !== void 0) await this.running.catch(() => []);
+		return this.run();
+	}
+	status() {
+		return {
+			watermark: this.watermark.status(),
+			consumers: this.entries().map((entry) => ({
+				consumerId: entry.consumer.consumerId,
+				thread: entry.thread,
+				appliedThrough: entry.consumer.appliedThrough,
+				trackedAbove: entry.consumer.trackedAbove,
+				...entry.blockedAt !== void 0 ? { blockedAt: entry.blockedAt } : {},
+				lastAdvanceUtcMs: entry.lastAdvanceUtcMs
+			})).concat(this.statusSources.flatMap((source) => source()))
+		};
+	}
+	ensureTimer() {
+		if (!this.started || this.stopped || this.timer !== void 0) return;
+		if (this.fixed.length === 0 && this.sources.length === 0) return;
+		const timer = setInterval(() => {
+			if (this.running !== void 0) return;
+			this.run().catch((error) => {
+				this.logger.error("catch-up tick failed: @error", error);
+			});
+		}, this.config.intervalMs);
+		if (typeof timer === "object" && "unref" in timer) timer.unref();
+		this.timer = timer;
+	}
+	run() {
+		const run = this.tick().finally(() => {
+			if (this.running === run) this.running = void 0;
+		});
+		this.running = run;
+		return run;
+	}
+	async tick() {
+		const settled = await this.watermark.refresh();
+		await this.warnIfWatermarkHeld();
+		const entries = this.entries();
+		if (entries.length === 0) return [];
+		let lowest = Number.POSITIVE_INFINITY;
+		for (const entry of entries) lowest = Math.min(lowest, entry.consumer.appliedThrough);
+		const shared = await this.readPage(lowest, settled);
+		const results = [];
+		for (const entry of entries) {
+			if (this.stopped) break;
+			const result = await this.sweepOne(entry, shared, settled);
+			if (result !== void 0) results.push(result);
+		}
+		return results;
+	}
+	entries() {
+		const entries = [...this.fixed];
+		const seen = new Set(entries.map((entry) => entry.consumer));
+		for (const source of this.sources) for (const consumer of source.consumers()) {
+			if (seen.has(consumer)) continue;
+			seen.add(consumer);
+			let entry = this.sourced.get(consumer);
+			if (entry === void 0) {
+				entry = {
+					consumer,
+					thread: source.thread,
+					lastAdvanceUtcMs: Date.now(),
+					warnedHeld: false
+				};
+				this.sourced.set(consumer, entry);
+			}
+			entries.push(entry);
+		}
+		return entries;
+	}
+	async sweepOne(entry, shared, settled) {
+		const applied = entry.consumer.appliedThrough;
+		let page = shared;
+		if (applied >= shared.bound && shared.bound < settled) try {
+			page = await this.readPage(applied, settled);
+		} catch (error) {
+			this.logger.error("@consumer catch-up could not read present ordinals: @error", entry.consumer.consumerId, error);
+			return;
+		}
+		let result;
+		try {
+			result = await entry.consumer.sweep(page.bound, page.present);
+		} catch (error) {
+			this.logger.error("@consumer catch-up sweep failed: @error", entry.consumer.consumerId, error);
+			return;
+		}
+		if (result.to > result.from) {
+			entry.lastAdvanceUtcMs = Date.now();
+			entry.warnedHeld = false;
+		}
+		entry.blockedAt = result.blockedAt;
+		this.warnIfConsumerHeld(entry, settled);
+		if (result.to > result.from || result.replayed > 0 || result.blockedAt !== void 0) this.hooks.onSwept(result, entry.thread);
+		return result;
+	}
+	async warnIfWatermarkHeld() {
+		const status = this.watermark.status();
+		const since = status.stalledSinceUtcMs;
+		if (since === void 0) {
+			this.warnedStallSince = void 0;
+			return;
+		}
+		const heldMs = Date.now() - since;
+		if (heldMs < this.config.stuckWarnMs || this.warnedStallSince === since) return;
+		this.warnedStallSince = since;
+		const sessions = await this.hooks.describeSessions(status.waitingOn).catch(() => []);
+		this.logger.warn("settled watermark held at @settled (head @head) for @ms ms, waiting on xid @xids: @sessions", status.settledThrough, status.head, heldMs, status.waitingOn.join(", "), sessions.map((session) => `pid ${session.pid} ${session.applicationName} ${session.state} since ${session.xactStart ?? "?"}`).join("; "));
+	}
+	warnIfConsumerHeld(entry, settled) {
+		const applied = entry.consumer.appliedThrough;
+		if (applied >= settled || entry.warnedHeld) return;
+		const heldMs = Date.now() - entry.lastAdvanceUtcMs;
+		if (heldMs < this.config.stuckWarnMs) return;
+		entry.warnedHeld = true;
+		this.logger.warn("@consumer cursor held at @applied for @ms ms (settled @settled)", entry.consumer.consumerId, applied, heldMs, settled);
+	}
+	async readPage(after, settled) {
+		const limit = this.config.sweepPageSize;
+		const present = await this.operationIndex.getOrdinalsInRange(after, settled, limit);
+		return {
+			bound: present.length >= limit ? present[present.length - 1] : settled,
+			present
+		};
+	}
+};
+//#endregion
+//#region src/catch-up/types.ts
+const defaultCatchUpConfig = {
+	intervalMs: 2e3,
+	stuckWarnMs: 6e4,
+	maxTrackedAboveCursor: 1e5,
+	sweepPageSize: 1e4
+};
+//#endregion
+//#region src/catch-up/contiguous-cursor.ts
+/** In memory, per consumer; tracks ordinals above appliedThrough only. */
+var ContiguousCursor = class {
+	through;
+	limit;
+	claimed = /* @__PURE__ */ new Set();
+	applied = /* @__PURE__ */ new Set();
+	constructor(through, limit) {
+		this.through = through;
+		this.limit = limit;
+	}
+	setLimit(limit) {
+		this.limit = limit;
+	}
+	/** Every present ordinal at or below this has been applied. */
+	get appliedThrough() {
+		return this.through;
+	}
+	get trackedAbove() {
+		return this.claimed.size + this.applied.size;
+	}
+	/** Ordinals the caller must apply; the rest are already covered. */
+	claim(ordinals) {
+		const mine = /* @__PURE__ */ new Set();
+		for (const ordinal of ordinals) {
+			if (ordinal <= this.through) continue;
+			if (this.claimed.has(ordinal) || this.applied.has(ordinal)) continue;
+			this.claimed.add(ordinal);
+			mine.add(ordinal);
+		}
+		return mine;
+	}
+	/** Releases claims; committed ones join the applied set. */
+	settle(ordinals, committed) {
+		for (const ordinal of ordinals) {
+			if (!this.claimed.delete(ordinal)) continue;
+			if (committed && ordinal > this.through) this.applied.add(ordinal);
+		}
+	}
+	/** Past the limit, clears the applied set; the next sweep re-applies it. */
+	enforceLimit() {
+		if (this.claimed.size + this.applied.size > this.limit) this.applied.clear();
+	}
+	/** Present ordinals above the cursor that no path has claimed or applied. */
+	missing(present) {
+		return present.filter((ordinal) => ordinal > this.through && !this.claimed.has(ordinal) && !this.applied.has(ordinal));
+	}
+	/** `settled`, held below the lowest claimed or unapplied ordinal. */
+	target(settled, present) {
+		let to = settled;
+		for (const ordinal of this.claimed) if (ordinal <= to) to = ordinal - 1;
+		for (const ordinal of present) {
+			if (ordinal <= this.through || ordinal > to) continue;
+			if (!this.applied.has(ordinal)) to = ordinal - 1;
+		}
+		return Math.max(to, this.through);
+	}
+	advance(to) {
+		if (to <= this.through) return;
+		this.through = to;
+		for (const ordinal of this.applied) if (ordinal <= to) this.applied.delete(ordinal);
+		for (const ordinal of this.claimed) if (ordinal <= to) this.claimed.delete(ordinal);
+	}
+};
+//#endregion
 //#region src/read-models/base-read-model.ts
 /** Small enough that a chunk's transaction rarely outlasts the yield deadline. */
 const DEFAULT_COMMIT_CHUNK_SIZE = 50;
@@ -30,26 +301,63 @@ function normalizeIndexingConfig(config) {
 		commitChunkSize: Math.max(1, Math.min(Math.floor(config.commitChunkSize), Number.MAX_SAFE_INTEGER))
 	};
 }
-/**
-* Base class for read models that provides catch-up/rewind functionality.
-* Handles initialization, state tracking via ViewState table, and consistency tracking.
-* Subclasses override commitOperations() with their specific domain logic.
-*/
+function ordinalOf(item) {
+	return item.context.ordinal;
+}
+function isTracked(ordinal) {
+	return Number.isFinite(ordinal) && ordinal > 0;
+}
+function streamKeyOf(stream) {
+	return `${stream.documentId}\u0000${stream.scope}\u0000${stream.branch}`;
+}
+function groupByStream(items) {
+	const groups = /* @__PURE__ */ new Map();
+	for (const item of items) {
+		const { documentId, scope, branch } = item.context;
+		const key = streamKeyOf(item.context);
+		let group = groups.get(key);
+		if (group === void 0) {
+			group = {
+				documentId,
+				scope,
+				branch,
+				lowest: ordinalOf(item),
+				late: []
+			};
+			groups.set(key, group);
+		}
+		group.lowest = Math.min(group.lowest, ordinalOf(item));
+		group.late.push(item);
+	}
+	return [...groups.values()].sort((a, b) => a.lowest - b.lowest);
+}
+function mergeByOrdinal(...lists) {
+	const byOrdinal = /* @__PURE__ */ new Map();
+	for (const list of lists) for (const item of list) byOrdinal.set(ordinalOf(item), item);
+	return [...byOrdinal.values()].sort((a, b) => ordinalOf(a) - ordinalOf(b));
+}
+function errorMessage(error) {
+	return error instanceof Error ? error.message : String(error);
+}
+/** A read model whose cursor only sweeps and boot replay advance. */
 var BaseReadModel = class {
 	db;
 	operationIndex;
 	writeCache;
 	consistencyTracker;
 	config;
-	lastOrdinal = 0;
 	name;
 	indexing;
-	/**
-	* Lowest ordinal this model failed to commit and has not committed since, or
-	* zero when there is none. The stored cursor is held below it so replay from
-	* the cursor still reaches every operation the failed pass left out.
-	*/
-	uncommittedOrdinal = 0;
+	catchUpLogger;
+	cursor;
+	persisted = 0;
+	watermark;
+	maxTrackedAboveCursor = defaultCatchUpConfig.maxTrackedAboveCursor;
+	loggedFailure;
+	failedItem;
+	initialized = false;
+	liveInFlight = /* @__PURE__ */ new Map();
+	sweptListeners = /* @__PURE__ */ new Set();
 	constructor(db, operationIndex, writeCache, consistencyTracker, config) {
 		this.db = db;
 		this.operationIndex = operationIndex;
@@ -57,52 +365,131 @@ var BaseReadModel = class {
 		this.consistencyTracker = consistencyTracker;
 		this.config = config;
 		this.name = config.readModelId;
+		this.catchUpLogger = childLogger([
+			"reactor",
+			"read-model",
+			config.readModelId
+		]);
 		this.indexing = normalizeIndexingConfig(config.indexing ?? defaultReadModelIndexingConfig);
+		this.cursor = new ContiguousCursor(0, this.maxTrackedAboveCursor);
 	}
-	/**
-	* Initializes the read model by loading state and catching up on missed operations.
-	*/
+	get consumerId() {
+		return this.config.readModelId;
+	}
+	get appliedThrough() {
+		return this.cursor.appliedThrough;
+	}
+	get trackedAbove() {
+		return this.cursor.trackedAbove;
+	}
+	/** Shares a thread's watermark; otherwise boot probes through this db. */
+	attachCatchUp(watermark, maxTrackedAboveCursor) {
+		this.watermark = watermark;
+		this.maxTrackedAboveCursor = maxTrackedAboveCursor;
+		this.cursor.setLimit(maxTrackedAboveCursor);
+	}
+	/** Notified with the coordinates each sweep applied. */
+	onSwept(listener) {
+		this.sweptListeners.add(listener);
+		return () => {
+			this.sweptListeners.delete(listener);
+		};
+	}
+	/** A chunk that throws ends the replay; sweeps continue from below it. */
 	async init() {
-		const viewState = await this.loadState();
-		if (viewState !== void 0) this.lastOrdinal = viewState;
-		else await this.initializeState();
-		let page = await this.operationIndex.getSinceOrdinal(this.lastOrdinal);
-		while (page.results.length > 0) {
-			const ops = this.config.rebuildStateOnInit ? await this.rebuildStateForOperations(page.results) : page.results;
-			await this.indexOperations(ops);
-			if (!page.next) break;
-			page = await page.next();
+		if (this.initialized) {
+			await this.replayFromCursor();
+			return;
+		}
+		let stored = await this.loadState();
+		if (stored === void 0) {
+			if (this.config.startFrom === "head") {
+				const watermark = this.settledWatermark();
+				const settled = await watermark.refresh();
+				const head = Math.max(settled, watermark.status().head);
+				await this.initializeState(head);
+				this.resetCursor(head);
+				this.initialized = true;
+				return;
+			}
+			await this.initializeState(0);
+			stored = 0;
 		}
+		this.resetCursor(stored);
+		this.initialized = true;
+		await this.replayFromCursor();
 	}
-	/**
-	* Commits the batch in chunks, yielding between them with no transaction
-	* open. A chunk that throws leaves the earlier chunks committed, so the pass
-	* saves the cursor for that prefix and parks it below the operation it could
-	* not commit before rethrowing.
-	*/
-	async indexOperations(items) {
-		if (items.length === 0) return;
-		const { commitChunkSize, yieldDeadlineMs } = this.indexing;
-		let lastYield = performance.now();
-		let committed = 0;
-		for (let start = 0; start < items.length; start += commitChunkSize) {
-			if (start > 0 && performance.now() - lastYield > yieldDeadlineMs) {
-				await yieldToMain();
-				lastYield = performance.now();
+	/** The live path: never moves the cursor. */
+	indexOperations(items) {
+		if (items.length === 0) return Promise.resolve();
+		return this.reserveOperations(items).apply();
+	}
+	/** Holds a queued batch as live, so a sweep neither takes nor passes it. */
+	reserveOperations(items) {
+		const cursor = this.cursor;
+		const owned = this.claimLive(items);
+		const unmark = this.markLive(owned);
+		let done = false;
+		return {
+			apply: () => {
+				if (done) return Promise.resolve();
+				done = true;
+				if (cursor !== this.cursor) {
+					unmark();
+					return this.indexOperations(items);
+				}
+				if (owned.length === 0) {
+					unmark();
+					return Promise.resolve();
+				}
+				return this.applyChunked(owned, unmark);
+			},
+			release: () => {
+				if (done) return;
+				done = true;
+				cursor.settle(owned.map(ordinalOf), false);
+				unmark();
 			}
-			const chunk = items.slice(start, start + commitChunkSize);
+		};
+	}
+	async sweep(settledThrough, present, signal) {
+		const startedAt = performance.now();
+		const from = this.cursor.appliedThrough;
+		const range = present.filter((ordinal) => ordinal > from && ordinal <= settledThrough);
+		const mine = this.cursor.claim(this.cursor.missing(range));
+		let replayed = 0;
+		let reapplied = 0;
+		let blockedAt;
+		if (mine.size > 0) {
+			let late;
 			try {
-				await this.commitOperations(chunk);
+				late = await this.operationIndex.getByOrdinals([...mine], signal);
 			} catch (error) {
-				this.park(items, committed);
-				await this.recordCommittedPrefix(items.slice(0, committed));
+				this.cursor.settle(mine, false);
 				throw error;
 			}
-			committed += chunk.length;
+			const found = new Set(late.map(ordinalOf));
+			this.cursor.settle([...mine].filter((ordinal) => !found.has(ordinal)), true);
+			for (const group of groupByStream(late)) {
+				const outcome = await this.sweepStream(group, signal);
+				replayed += outcome.replayed;
+				reapplied += outcome.reapplied;
+				blockedAt ??= outcome.blockedAt;
+			}
+			if (replayed > 0) this.catchUpLogger.warn("@consumer applied @n operations its live path never received: ordinals @first..@last", this.consumerId, replayed, Math.min(...found), Math.max(...found));
 		}
-		this.liftParkIfCommitted(items);
-		await this.persistCursor(items);
-		this.updateConsistencyTracker(items);
+		const to = this.cursor.target(settledThrough, range);
+		if (to > from) await this.moveCursor(to);
+		this.cursor.enforceLimit();
+		return {
+			consumerId: this.consumerId,
+			from,
+			to: this.cursor.appliedThrough,
+			durationMs: performance.now() - startedAt,
+			replayed,
+			reapplied,
+			...blockedAt !== void 0 ? { blockedAt } : {}
+		};
 	}
 	/**
 	* Waits for the read model to reach the specified consistency level.
@@ -111,10 +498,14 @@ var BaseReadModel = class {
 		if (token.coordinates.length === 0) return;
 		await this.consistencyTracker.waitFor(token.coordinates, timeoutMs, signal);
 	}
+	/** The cursor, for subclasses: every present ordinal at or below it is applied. */
+	get lastOrdinal() {
+		return this.cursor.appliedThrough;
+	}
+	/** Called after the cursor advances. */
+	onCursorAdvanced(appliedThrough) {}
 	async commitOperations(items) {}
-	/**
-	* Rebuilds document state for each operation using the write cache.
-	*/
+	/** Rebuilds resultingState as the executor writes it: scopes plus header. */
 	async rebuildStateForOperations(operations) {
 		const result = [];
 		for (const op of operations) {
@@ -125,7 +516,10 @@ var BaseReadModel = class {
 				operation: op.operation,
 				context: {
 					...op.context,
-					resultingState: JSON.stringify(document)
+					resultingState: JSON.stringify({
+						...document.state,
+						header: document.header
+					})
 				}
 			});
 		}
@@ -141,29 +535,19 @@ var BaseReadModel = class {
 	/**
 	* Initializes the ViewState row for this read model.
 	*/
-	async initializeState() {
+	async initializeState(lastOrdinal = 0) {
 		await this.db.insertInto("ViewState").values({
 			readModelId: this.config.readModelId,
-			lastOrdinal: 0
-		}).execute();
-	}
-	/**
-	* Saves the last processed ordinal to the ViewState table.
-	*/
-	async saveState(trx, items) {
-		let maxOrdinal = 0;
-		for (const item of items) maxOrdinal = Math.max(maxOrdinal, item.context.ordinal);
-		const next = Math.max(this.lastOrdinal, maxOrdinal);
-		this.lastOrdinal = next;
-		await trx.updateTable("ViewState").set({
-			lastOrdinal: next,
-			lastOperationTimestamp: /* @__PURE__ */ new Date()
-		}).where("readModelId", "=", this.config.readModelId).execute();
+			lastOrdinal
+		}).onConflict((oc) => oc.column("readModelId").doNothing()).execute();
 	}
 	/**
 	* Updates the consistency tracker with the processed operations.
 	*/
 	updateConsistencyTracker(items) {
+		this.consistencyTracker.update(this.coordinatesOf(items));
+	}
+	coordinatesOf(items) {
 		const coordinates = [];
 		for (let i = 0; i < items.length; i++) {
 			const item = items[i];
@@ -174,64 +558,297 @@ var BaseReadModel = class {
 				operationIndex: item.operation.index
 			});
 		}
-		this.consistencyTracker.update(coordinates);
+		return coordinates;
 	}
-	/**
-	* Saves the cursor for the chunks that did commit before a later chunk threw.
-	* A failure here is swallowed: the cursor simply stays where the pass found
-	* it, which is equally safe, and the commit error is the one worth raising.
-	*/
-	async recordCommittedPrefix(prefix) {
-		if (prefix.length === 0) return;
+	settledWatermark() {
+		this.watermark ??= new SettledWatermark(createKyselyWatermarkProbe(this.db), this.catchUpLogger);
+		return this.watermark;
+	}
+	resetCursor(appliedThrough) {
+		this.cursor = new ContiguousCursor(appliedThrough, this.maxTrackedAboveCursor);
+		this.persisted = appliedThrough;
+	}
+	/** Items this pass must apply; untracked ordinals always apply. */
+	claimLive(items) {
+		const mine = this.cursor.claim(items.map(ordinalOf));
+		return items.filter((item) => {
+			const ordinal = ordinalOf(item);
+			return !isTracked(ordinal) || mine.has(ordinal);
+		});
+	}
+	/** Marks items in flight on their streams; returns the release. */
+	markLive(items) {
+		const keys = /* @__PURE__ */ new Set();
+		for (const item of items) {
+			if (!isTracked(ordinalOf(item))) continue;
+			const key = streamKeyOf(item.context);
+			let ordinals = this.liveInFlight.get(key);
+			if (ordinals === void 0) {
+				ordinals = /* @__PURE__ */ new Set();
+				this.liveInFlight.set(key, ordinals);
+			}
+			ordinals.add(ordinalOf(item));
+			keys.add(key);
+		}
+		return () => {
+			for (const item of items) this.liveInFlight.get(streamKeyOf(item.context))?.delete(ordinalOf(item));
+			for (const key of keys) if (this.liveInFlight.get(key)?.size === 0) this.liveInFlight.delete(key);
+		};
+	}
+	/** True while a live pass applies an earlier operation of the stream. */
+	liveBelow(group) {
+		const ordinals = this.liveInFlight.get(streamKeyOf(group));
+		if (ordinals === void 0) return false;
+		for (const ordinal of ordinals) if (ordinal < group.lowest) return true;
+		return false;
+	}
+	/** Commits in chunks, yielding with no transaction open. */
+	async applyChunked(items, release = this.markLive(items)) {
 		try {
-			await this.persistCursor(prefix);
-		} catch {
-			return;
+			const { commitChunkSize, yieldDeadlineMs } = this.indexing;
+			let lastYield = performance.now();
+			let committed = 0;
+			for (let start = 0; start < items.length; start += commitChunkSize) {
+				if (start > 0 && performance.now() - lastYield > yieldDeadlineMs) {
+					await yieldToMain();
+					lastYield = performance.now();
+				}
+				const chunk = items.slice(start, start + commitChunkSize);
+				try {
+					await this.commitOperations(chunk);
+				} catch (error) {
+					this.failedItem = chunk[0];
+					const prefix = items.slice(0, committed);
+					this.cursor.settle(prefix.map(ordinalOf), true);
+					this.cursor.settle(items.slice(committed).map(ordinalOf), false);
+					if (prefix.length > 0) this.updateConsistencyTracker(prefix);
+					throw error;
+				}
+				committed += chunk.length;
+			}
+			this.cursor.settle(items.map(ordinalOf), true);
+			this.cursor.enforceLimit();
+			this.updateConsistencyTracker(items);
+		} finally {
+			release();
 		}
-		this.updateConsistencyTracker(prefix);
 	}
-	/** Writes the cursor for the given items, never past a parked ordinal. */
-	async persistCursor(items) {
-		await this.db.transaction().execute(async (trx) => {
-			await this.saveState(trx, items);
-			await this.clampCursorToPark(trx);
-		});
+	async replayFromCursor() {
+		const settledAtBoot = await this.settledAtBoot();
+		let page = await this.operationIndex.getSinceOrdinal(this.cursor.appliedThrough);
+		while (page.results.length > 0) {
+			const ordinals = page.results.map(ordinalOf);
+			const failed = await this.replayPage(page.results);
+			const pageMax = Math.max(...ordinals);
+			const to = this.cursor.target(Math.min(settledAtBoot, pageMax), ordinals);
+			if (to > this.cursor.appliedThrough) await this.moveCursor(to);
+			if (failed || !page.next) break;
+			page = await page.next();
+		}
 	}
-	/**
-	* Holds the cursor written by {@link saveState}, which subclasses may
-	* override, below the lowest operation this model failed to commit.
-	*/
-	async clampCursorToPark(trx) {
-		if (this.uncommittedOrdinal === 0) return;
-		const ceiling = this.uncommittedOrdinal - 1;
-		if (this.lastOrdinal <= ceiling) return;
-		this.lastOrdinal = ceiling;
-		await trx.updateTable("ViewState").set({
-			lastOrdinal: ceiling,
+	/** Without a probe, boot replay applies but leaves the cursor to sweeps. */
+	async settledAtBoot() {
+		try {
+			return await this.settledWatermark().refresh();
+		} catch (error) {
+			this.catchUpLogger.warn("@consumer could not probe the settled watermark at boot; the cursor waits for a sweep: @error", this.consumerId, error);
+			return this.cursor.appliedThrough;
+		}
+	}
+	/** Returns true when a chunk failed, which ends the replay. */
+	async replayPage(results) {
+		const owned = this.claimLive(results);
+		if (owned.length === 0) return false;
+		const release = this.markLive(owned);
+		try {
+			return await this.replayOwned(owned);
+		} finally {
+			release();
+		}
+	}
+	async replayOwned(owned) {
+		let rebuilt;
+		try {
+			rebuilt = await this.rebuildIfConfigured(owned);
+		} catch (error) {
+			this.cursor.settle(owned.map(ordinalOf), false);
+			this.block(owned[0], error);
+			return true;
+		}
+		this.cursor.settle(rebuilt.absent, true);
+		if (rebuilt.items.length === 0) return false;
+		try {
+			await this.applyChunked(rebuilt.items);
+		} catch (error) {
+			this.block(this.failedItem ?? rebuilt.items[0], error);
+			return true;
+		}
+		return false;
+	}
+	async sweepStream(group, signal) {
+		const owned = group.late.map(ordinalOf);
+		if (this.liveBelow(group)) {
+			this.cursor.settle(owned, false);
+			return {
+				replayed: 0,
+				reapplied: 0
+			};
+		}
+		const ownedSet = new Set(owned);
+		let items = group.late;
+		if (this.config.replayStreamSuffix ?? true) {
+			let suffix;
+			try {
+				suffix = await this.operationIndex.getStreamAfter(group, group.lowest, signal);
+			} catch (error) {
+				this.cursor.settle(owned, false);
+				return {
+					replayed: 0,
+					reapplied: 0,
+					blockedAt: this.block(group.late[0], error)
+				};
+			}
+			items = mergeByOrdinal(group.late, suffix);
+		}
+		let rebuilt;
+		try {
+			rebuilt = await this.rebuildIfConfigured(items);
+		} catch (error) {
+			this.cursor.settle(owned, false);
+			return {
+				replayed: 0,
+				reapplied: 0,
+				blockedAt: this.block(items[0], error)
+			};
+		}
+		try {
+			await this.commitOperations(rebuilt.items);
+		} catch (error) {
+			this.cursor.settle(owned, false);
+			return {
+				replayed: 0,
+				reapplied: 0,
+				blockedAt: this.block(rebuilt.items[0] ?? items[0], error)
+			};
+		}
+		this.cursor.settle(owned, true);
+		this.updateConsistencyTracker(rebuilt.items);
+		this.notifySwept(rebuilt.items);
+		let replayed = 0;
+		for (const item of rebuilt.items) if (ownedSet.has(ordinalOf(item))) replayed++;
+		return {
+			replayed,
+			reapplied: rebuilt.items.length - replayed
+		};
+	}
+	/** A document gone from the write cache is absent, not a failure. */
+	async rebuildIfConfigured(items) {
+		if (!this.config.rebuildStateOnInit) return {
+			items,
+			absent: []
+		};
+		const rebuilt = [];
+		const absent = [];
+		for (const item of items) {
+			let result;
+			try {
+				result = await this.rebuildStateForOperations([item]);
+			} catch (error) {
+				if (!(error instanceof DocumentNotFoundError)) throw error;
+				this.catchUpLogger.warn("@consumer dropped ordinal @ordinal: document @documentId is gone", this.consumerId, ordinalOf(item), item.context.documentId);
+				absent.push(ordinalOf(item));
+				continue;
+			}
+			rebuilt.push(...result);
+		}
+		return {
+			items: rebuilt,
+			absent
+		};
+	}
+	block(item, error) {
+		const blockedAt = {
+			ordinal: ordinalOf(item),
+			documentId: item.context.documentId,
+			scope: item.context.scope,
+			branch: item.context.branch,
+			type: item.operation.action.type,
+			error: errorMessage(error)
+		};
+		if (this.loggedFailure !== blockedAt.ordinal) {
+			this.loggedFailure = blockedAt.ordinal;
+			this.catchUpLogger.error("@consumer cursor held at @applied: ordinal @ordinal (@documentId/@scope/@branch, @type) failed: @error", this.consumerId, this.cursor.appliedThrough, blockedAt.ordinal, blockedAt.documentId, blockedAt.scope, blockedAt.branch, blockedAt.type, blockedAt.error);
+		}
+		return blockedAt;
+	}
+	notifySwept(items) {
+		if (this.sweptListeners.size === 0 || items.length === 0) return;
+		const coordinates = this.coordinatesOf(items);
+		for (const listener of this.sweptListeners) listener(coordinates);
+	}
+	/** Compare-and-set against the value this process last wrote. */
+	async moveCursor(to) {
+		if (to <= this.persisted) {
+			this.advanceCursor(to);
+			return;
+		}
+		const expected = this.persisted;
+		const result = await this.db.updateTable("ViewState").set({
+			lastOrdinal: to,
 			lastOperationTimestamp: /* @__PURE__ */ new Date()
-		}).where("readModelId", "=", this.config.readModelId).execute();
-	}
-	/** Remembers the lowest ordinal the failed pass left uncommitted. */
-	park(items, committed) {
-		let lowest = 0;
-		for (let i = committed; i < items.length; i++) {
-			const ordinal = items[i].context.ordinal;
-			if (lowest === 0 || ordinal < lowest) lowest = ordinal;
+		}).where("readModelId", "=", this.config.readModelId).where("lastOrdinal", "=", expected).executeTakeFirst();
+		if (Number(result.numUpdatedRows) > 0) {
+			this.persisted = to;
+			this.advanceCursor(to);
+			return;
+		}
+		const stored = await this.loadState();
+		if (stored === void 0) {
+			await this.initializeState(0);
+			this.lowered(0);
+			return;
 		}
-		if (lowest === 0) return;
-		if (this.uncommittedOrdinal === 0 || lowest < this.uncommittedOrdinal) this.uncommittedOrdinal = lowest;
-	}
-	/** The park lifts once a later pass commits the operation that failed. */
-	liftParkIfCommitted(items) {
-		if (this.uncommittedOrdinal === 0) return;
-		for (const item of items) if (item.context.ordinal === this.uncommittedOrdinal) {
-			this.uncommittedOrdinal = 0;
+		if (stored < this.cursor.appliedThrough) {
+			this.lowered(stored);
 			return;
 		}
+		this.persisted = stored;
+		this.advanceCursor(Math.min(to, stored));
+	}
+	advanceCursor(to) {
+		const before = this.cursor.appliedThrough;
+		this.cursor.advance(to);
+		if (this.cursor.appliedThrough > before) this.onCursorAdvanced(this.cursor.appliedThrough);
+	}
+	lowered(to) {
+		this.catchUpLogger.info("@consumer cursor lowered externally from @old to @new; replaying", this.consumerId, this.cursor.appliedThrough, to);
+		this.resetCursor(to);
 	}
 };
 //#endregion
 //#region src/read-models/coordinator.ts
+/** Claims a batch for every model that supports it, as the batch queues. */
+function reserveBatch(readModels, operations, logger) {
+	const reservations = /* @__PURE__ */ new Map();
+	for (const readModel of readModels) {
+		if (readModel.reserveOperations === void 0) continue;
+		try {
+			reservations.set(readModel, readModel.reserveOperations(operations));
+		} catch (error) {
+			logger.error("Read model @name could not reserve a batch: @Error", readModel.name, error);
+		}
+	}
+	return reservations;
+}
+/** Applies the model's reservation, or indexes a model registered since. */
+function indexReserved(readModel, operations, reservations) {
+	const reservation = reservations.get(readModel);
+	return reservation ? reservation.apply() : readModel.indexOperations(operations);
+}
+/** Frees every claim a run did not apply, so a sweep can take it. */
+function releaseBatch(reservations) {
+	for (const reservation of reservations.values()) reservation.release();
+}
 /**
 * Coordinates read model synchronization by listening to operation write events
 * and updating all registered read models on per-`documentId:scope:branch`
@@ -284,6 +901,9 @@ var ReadModelCoordinator = class {
 	getChainDepth() {
 		return this.chains.size;
 	}
+	indexedReadModels() {
+		return [...this.preReady, ...this.postReady];
+	}
 	addReadModel(readModel, stage) {
 		if (this.readModels.some(({ name }) => name === readModel.name)) throw new Error(`Read model "${readModel.name}" is already registered`);
 		if (stage === "pre_ready") this.preReady.push(readModel);
@@ -297,7 +917,9 @@ var ReadModelCoordinator = class {
 		}
 		const enqueuedAt = performance.now();
 		const key = this.queueKeyFor(event);
-		const current = (this.chains.get(key) ?? Promise.resolve()).then(() => this.runChain(event, enqueuedAt));
+		const reservations = reserveBatch(this.readModels, event.operations, this.logger);
+		const run = () => this.runChain(event, enqueuedAt, reservations);
+		const current = (this.chains.get(key) ?? Promise.resolve()).then(run, run);
 		this.chains.set(key, current);
 		current.finally(() => {
 			if (this.chains.get(key) === current) this.chains.delete(key);
@@ -322,11 +944,18 @@ var ReadModelCoordinator = class {
 			postReadyDurationMs: 0
 		});
 	}
-	async runChain(event, enqueuedAt) {
+	async runChain(event, enqueuedAt, reservations) {
+		try {
+			await this.runBatch(event, enqueuedAt, reservations);
+		} finally {
+			releaseBatch(reservations);
+		}
+	}
+	async runBatch(event, enqueuedAt, reservations) {
 		const chainWaitDurationMs = performance.now() - enqueuedAt;
 		const preReadyStart = performance.now();
 		try {
-			await Promise.all(this.preReady.map((readModel) => this.indexWithTiming(readModel, "pre_ready", event)));
+			await Promise.all(this.preReady.map((readModel) => this.indexWithTiming(readModel, "pre_ready", event, reservations)));
 		} catch (error) {
 			this.logger.error("Pre-ready read model indexing failed for job @jobId: @Error", { jobId: event.jobId }, error);
 		}
@@ -344,7 +973,7 @@ var ReadModelCoordinator = class {
 		const emitDurationMs = performance.now() - emitStart;
 		const postReadyStart = performance.now();
 		try {
-			await Promise.all(this.postReady.map((readModel) => this.indexWithTiming(readModel, "post_ready", event)));
+			await Promise.all(this.postReady.map((readModel) => this.indexWithTiming(readModel, "post_ready", event, reservations)));
 		} catch (error) {
 			this.logger.error("Post-ready read model indexing failed for job @jobId: @Error", { jobId: event.jobId }, error);
 		}
@@ -358,11 +987,11 @@ var ReadModelCoordinator = class {
 			postReadyDurationMs
 		});
 	}
-	async indexWithTiming(readModel, stage, event) {
+	async indexWithTiming(readModel, stage, event, reservations) {
 		const start = performance.now();
 		let success = false;
 		try {
-			await readModel.indexOperations(event.operations);
+			await indexReserved(readModel, event.operations, reservations);
 			success = true;
 		} finally {
 			this.emitReadModelIndexed({
@@ -420,7 +1049,8 @@ var KyselyDocumentView = class extends BaseReadModel {
 		super(db, operationIndex, writeCache, consistencyTracker, {
 			readModelId: DOCUMENT_VIEW_READ_MODEL,
 			rebuildStateOnInit: true,
-			indexing
+			indexing,
+			replayStreamSuffix: false
 		});
 		this.operationStore = operationStore;
 		this.deletedDocumentRead = deletedDocumentRead;
@@ -436,10 +1066,9 @@ var KyselyDocumentView = class extends BaseReadModel {
 	* echoes may be stale. All other action types index only header and their
 	* own scope.
 	*
-	* The header row is the one row every scope's chain writes, so it accepts a
-	* write only from an operation whose global ordinal is at least the one the
-	* row already carries. Without that, a chunked pass that started earlier
-	* reverts a concurrent rename with the stale echo its later chunks carry.
+	* Every row accepts a write only from an operation whose global ordinal is
+	* at least the one the row already carries, so an older duplicate from a
+	* sweep, boot replay or chunk interleave never rolls a scope back.
 	*/
 	async commitOperations(items) {
 		await this._db.transaction().execute(async (trx) => {
@@ -466,9 +1095,9 @@ var KyselyDocumentView = class extends BaseReadModel {
 						lastOperationHash: hash,
 						lastOperationOrdinal: ordinal,
 						lastUpdatedAt: now
-					}).where("documentId", "=", documentId).where("branch", "=", branch).execute();
+					}).where("documentId", "=", documentId).where("branch", "=", branch).where("lastOperationOrdinal", "<=", ordinal).execute();
 					const deletedDocumentScope = fullState?.document;
-					if (deletedDocumentScope !== void 0) await trx.updateTable("DocumentSnapshot").set({ content: deletedDocumentScope }).where("documentId", "=", documentId).where("branch", "=", branch).where("scope", "=", "document").execute();
+					if (deletedDocumentScope !== void 0) await trx.updateTable("DocumentSnapshot").set({ content: deletedDocumentScope }).where("documentId", "=", documentId).where("branch", "=", branch).where("scope", "=", "document").where("lastOperationOrdinal", "<=", ordinal).execute();
 					await trx.deleteFrom("SlugMapping").where("documentId", "=", documentId).where("branch", "=", branch).execute();
 					continue;
 				}
@@ -494,7 +1123,7 @@ var KyselyDocumentView = class extends BaseReadModel {
 						"snapshotVersion",
 						"lastOperationOrdinal"
 					]).$if(needsExistingContent, (qb) => qb.select("content")).where("documentId", "=", documentId).where("scope", "=", scopeName).where("branch", "=", branch).executeTakeFirst();
-					if (scopeName === "header" && existingSnapshot !== void 0 && existingSnapshot.lastOperationOrdinal > ordinal) continue;
+					if (existingSnapshot !== void 0 && existingSnapshot.lastOperationOrdinal > ordinal) continue;
 					const newState = typeof scopeState === "object" && scopeState !== null ? scopeState : {};
 					let slug = existingSnapshot?.slug ?? null;
 					let name = existingSnapshot?.name ?? null;
@@ -527,7 +1156,7 @@ var KyselyDocumentView = class extends BaseReadModel {
 						content: newState,
 						slug,
 						name
-					}).where("documentId", "=", documentId).where("scope", "=", scopeName).where("branch", "=", branch).$if(scopeName === "header", (qb) => qb.where("lastOperationOrdinal", "<=", ordinal)).execute();
+					}).where("documentId", "=", documentId).where("scope", "=", scopeName).where("branch", "=", branch).where("lastOperationOrdinal", "<=", ordinal).execute();
 					else {
 						const snapshot = {
 							id: v4(),
@@ -1103,6 +1732,6 @@ var KyselyDocumentIndexer = class extends BaseReadModel {
 	}
 };
 //#endregion
-export { KyselyDocumentView as a, ReadModelCoordinator as c, DEFAULT_READ_MODEL_YIELD_DEADLINE_MS as d, defaultReadModelIndexingConfig as f, DeletedDocumentRead as i, BaseReadModel as l, ConsistencyTracker as n, DOCUMENT_INDEXER_READ_MODEL as o, unchunkedReadModelIndexingConfig as p, makeConsistencyKey as r, DOCUMENT_VIEW_READ_MODEL as s, KyselyDocumentIndexer as t, DEFAULT_COMMIT_CHUNK_SIZE as u };
+export { ContiguousCursor as _, KyselyDocumentView as a, isCatchUpConsumer as b, ReadModelCoordinator as c, reserveBatch as d, BaseReadModel as f, unchunkedReadModelIndexingConfig as g, defaultReadModelIndexingConfig as h, DeletedDocumentRead as i, indexReserved as l, DEFAULT_READ_MODEL_YIELD_DEADLINE_MS as m, ConsistencyTracker as n, DOCUMENT_INDEXER_READ_MODEL as o, DEFAULT_COMMIT_CHUNK_SIZE as p, makeConsistencyKey as r, DOCUMENT_VIEW_READ_MODEL as s, KyselyDocumentIndexer as t, releaseBatch as u, defaultCatchUpConfig as v, CatchUpScheduler as y };
 
-//# sourceMappingURL=document-indexer-BVTGbEB0.js.map
+//# sourceMappingURL=document-indexer-I6i_YzsR.js.map